From 5690fd0d3304f378754b23b098bd7cb5f4aa1976 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 14:38:21 +0200 Subject: [PATCH 0001/3719] initial commit with latest version extracted from git-python --- .gitignore | 1 + __init__.py | 6 + db.py | 341 +++++++++++++++++++++++++++++++++ fun.py | 115 ++++++++++++ stream.py | 445 ++++++++++++++++++++++++++++++++++++++++++++ test/__init__.py | 1 + test/lib.py | 60 ++++++ test/test_db.py | 90 +++++++++ test/test_stream.py | 172 +++++++++++++++++ test/test_utils.py | 15 ++ utils.py | 38 ++++ 11 files changed, 1284 insertions(+) create mode 100644 .gitignore create mode 100644 __init__.py create mode 100644 db.py create mode 100644 fun.py create mode 100644 stream.py create mode 100644 test/__init__.py create mode 100644 test/lib.py create mode 100644 test/test_db.py create mode 100644 test/test_stream.py create mode 100644 test/test_utils.py create mode 100644 utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..0d20b6487 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/__init__.py b/__init__.py new file mode 100644 index 000000000..5789d7eb7 --- /dev/null +++ b/__init__.py @@ -0,0 +1,6 @@ +"""Initialize the object database module""" + +# default imports +from db import * +from stream import * + diff --git a/db.py b/db.py new file mode 100644 index 000000000..5d3cc6a3f --- /dev/null +++ b/db.py @@ -0,0 +1,341 @@ +"""Contains implementations of database retrieveing objects""" +from git.utils import IndexFileSHA1Writer +from git.errors import ( + InvalidDBRoot, + BadObject, + BadObjectType + ) + +from stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + Sha1Writer, + OStream, + OInfo + ) + +from utils import ( + ENOENT, + to_hex_sha, + exists, + hex_to_bin, + isdir, + mkdir, + rename, + dirname, + join + ) + +from fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) + +import tempfile +import mmap +import os + + +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', + 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) + +class ObjectDBR(object): + """Defines an interface for object database lookup. + Objects are identified either by hex-sha (40 bytes) or + by sha (20 bytes)""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + :return: True if the object identified by the given 40 byte hexsha or 20 bytes + binary sha is contained in the database + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + """ :return: OInfo instance + :param sha: 40 bytes hexsha or 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info_async(self, input_channel): + """Retrieve information of a multitude of objects asynchronously + :param input_channel: Channel yielding the sha's of the objects of interest + :return: Channel yielding OInfo|InvalidOInfo, in any order""" + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + """:return: OStream instance + :param sha: 40 bytes hexsha or 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream_async(self, input_channel): + """Retrieve the OStream of multiple objects + :param input_channel: see ``info`` + :param max_threads: see ``ObjectDBW.store`` + :return: Channel yielding OStream|InvalidOStream instances in any order""" + raise NotImplementedError("To be implemented in subclass") + + #} END query interface + +class ObjectDBW(object): + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """Adjusts the stream to which all data should be sent when storing new objects + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """:return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + def store_async(self, input_channel): + """Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as + they are computed. + + :return: Channel yielding your IStream which served as input, in any order. + The IStreams sha will be set to the sha it received during the process, + or its error attribute will be set to the exception informing about the error. + :param input_channel: Channel yielding IStream instance. + As the same instances will be used in the output channel, you can create a map + between the id(istream) -> istream + :note:As some ODB implementations implement this operation as atomic, they might + abort the whole operation if one item could not be processed. Hence check how + many items have actually been produced.""" + raise NotImplementedError("To be implemented in subclass") + + #} END edit interface + + +class FileDBBase(object): + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + :note: The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super(FileDBBase, self).__init__() + self._root_path = root_path + + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, rela_path) + #} END interface + + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(to_hex_sha(sha))) + try: + fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + raise BadObject(to_hex_sha(sha)) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(to_hex_sha(sha)) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(to_hex_sha(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + writer = FDCompressedSha1Writer(fd) + # END handle custom writer + + try: + try: + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + + sha = istream.sha or writer.sha(as_hex=True) + + if tmp_path: + obj_path = self.db_path(self.object_path(sha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + rename(tmp_path, obj_path) + # END handle dry_run + + istream.sha = sha + return istream + + +class PackedDB(FileDBBase, ObjectDBR): + """A database operating on a set of object packs""" + + +class CompoundDB(ObjectDBR): + """A database which delegates calls to sub-databases""" + + +class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" + + +#class GitObjectDB(CompoundDB, ObjectDBW): +class GitObjectDB(LooseObjectDB): + """A database representing the default git object store, which includes loose + objects, pack files and an alternates file + + It will create objects only in the loose object database. + :note: for now, we use the git command to do all the lookup, just until he + have packs and the other implementations + """ + def __init__(self, root_path, git): + """Initialize this instance with the root and a git command""" + super(GitObjectDB, self).__init__(root_path) + self._git = git + + def info(self, sha): + t = self._git.get_object_header(sha) + return OInfo(*t) + + def stream(self, sha): + """For now, all lookup is done by git itself""" + t = self._git.stream_object_data(sha) + return OStream(*t) + diff --git a/fun.py b/fun.py new file mode 100644 index 000000000..3321a8ea4 --- /dev/null +++ b/fun.py @@ -0,0 +1,115 @@ +"""Contains basic c-functions which usually contain performance critical code +Keeping this code separate from the beginning makes it easier to out-source +it into c later, if required""" + +from git.errors import ( + BadObjectType + ) + +import zlib +decompressobj = zlib.decompressobj + + +# INVARIANTS +type_id_to_type_map = { + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag" + } + +# used when dealing with larger streams +chunk_size = 1000*1000 + +__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', + 'write_object' ) + +#{ Routines + +def is_loose_object(m): + """:return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 + +def loose_object_header_info(m): + """:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find("\0")].split(" ") + return type_name, int(size) + +def object_header_info(m): + """:return: tuple(type_string, uncompressed_size_in_bytes + :param mmap: mapped memory map. It will be + seeked to the actual start of the object contents, which can be used + to initialize a zlib decompress object. + :note: This routine can only handle new-style objects which are assumably contained + in packs + """ + assert not is_loose_object(m), "Use loose_object_header_info instead" + + c = b0 # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = ord(m[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + + # finally seek the map to the start of the data stream + m.seek(i) + try: + return (type_id_to_type_map[type_id], size) + except KeyError: + # invalid object type - we could try to be smart now and decode part + # of the stream to get the info, problem is that we had trouble finding + # the exact start of the content stream + raise BadObjectType(type_id) + # END handle exceptions + +def write_object(type, size, read, write, chunk_size=chunk_size): + """Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write("%s %i\0" % (type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw + +def stream_copy(read, write, size, chunk_size): + """Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + :note: its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size-dbw) + data_len = write(read(cs)) + dbw += data_len + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + + +#} END routines diff --git a/stream.py b/stream.py new file mode 100644 index 000000000..da97cf5b6 --- /dev/null +++ b/stream.py @@ -0,0 +1,445 @@ +import zlib +from cStringIO import StringIO +from git.utils import make_sha +import errno + +from utils import ( + to_hex_sha, + to_bin_sha, + write, + close + ) + +__all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', + 'DecompressMemMapReader', 'FDCompressedSha1Writer') + + +# ZLIB configuration +# used when compressing objects - 1 to 9 ( slowest ) +Z_BEST_SPEED = 1 + + +#{ ODB Bases + +class OInfo(tuple): + """Carries information about an object in an ODB, provdiing information + about the sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.sha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def sha(self): + return self[0] + + @property + def type(self): + return self[1] + + @property + def size(self): + return self[2] + #} END interface + + +class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + #} END stream reader interface + + +class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_sha(self, sha): + self[0] = sha + + def _sha(self): + return self[0] + + sha = property(_sha, _set_sha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def sha(self): + return self[0] + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases + + +#{ RO Streams + +class DecompressMemMapReader(object): + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + :note: The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, thats all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close') + + max_read_size = 512*1024 # currently unused + + def __init__(self, m, close_on_deletion, size): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._close = close_on_deletion # close the memmap on deletion ? + + def __del__(self): + if self._close: + self._m.close() + # END handle resource freeing + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + :return: parsed type_string, size""" + # read header + maxb = 512 # should really be enough, cgit uses 8192 I believe + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find("\0") + type, size = hdr[:hdrend].split(" ") + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 # count terminating \0 + self._buf = StringIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + return type, size + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + type, size = inst._parse_header_info() + return type, size, inst + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return str() + # END handle depletion + + # protect from memory peaks + # If he tries to read large chunks, our memory patterns get really bad + # as we end up copying a possibly huge chunk from our memory map right into + # memory. This might not even be possible. Nonetheless, try to dampen the + # effect a bit by reading in chunks, returning a huge string in the end. + # Our performance now depends on StringIO. This way we don't need two large + # buffers in peak times, but only one large one in the end which is + # the return buffer + # NO: We don't do it - if the user thinks its best, he is right. If he + # has trouble, he will start reading in chunks. According to our tests + # its still faster if we read 10 Mb at once instead of chunking it. + + # if size > self.max_read_size: + # sio = StringIO() + # while size: + # read_size = min(self.max_read_size, size) + # data = self.read(read_size) + # sio.write(data) + # size -= len(data) + # if len(data) < read_size: + # break + # # END data loop + # sio.seek(0) + # return sio.getvalue() + # # END handle maxread + # + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = str() + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes durin g + # the next read. + tail = self._zip.unconsumed_tail + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + size + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + size + # END handle tail + + + # if window is too small, make it larger so zip can decompress something + win_size = self._cwe - self._cws + if win_size < 8: + self._cwe = self._cws + 8 + # END adjust winsize + indata = self._m[self._cws:self._cwe] # another copy ... :( + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + + dcompdat = self._zip.decompress(indata, size) + + self._br += len(dcompdat) + if dat: + dcompdat = dat + dcompdat + + return dcompdat + +#} END RO streams + + +#{ W Streams + +class Sha1Writer(object): + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha("") + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex = False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface + +class FDCompressedSha1Writer(Sha1Writer): + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + :note: operates on raw file descriptors + :note: for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super(FDCompressedSha1Writer, self).__init__() + self.fd = fd + self.zip = zlib.compressobj(Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): + raise self.exc + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface + +#} END W streams diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test/lib.py b/test/lib.py new file mode 100644 index 000000000..d51997488 --- /dev/null +++ b/test/lib.py @@ -0,0 +1,60 @@ +"""Utilities used in ODB testing""" +from git.odb import ( + OStream, + ) +from git.odb.stream import Sha1Writer + +import zlib +from cStringIO import StringIO + +#{ Stream Utilities + +class DummyStream(object): + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read + + +class DeriveTest(OStream): + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg + + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(1) # fastest + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + +#} END stream utilitiess + diff --git a/test/test_db.py b/test/test_db.py new file mode 100644 index 000000000..35ba86802 --- /dev/null +++ b/test/test_db.py @@ -0,0 +1,90 @@ +"""Test for object db""" +from test.testlib import * +from lib import ZippedStoreShaWriter + +from git.odb import * +from git.odb.stream import Sha1Writer +from git import Blob +from git.errors import BadObject + + +from cStringIO import StringIO +import os + +class TestDB(TestBase): + """Test the different db class implementations""" + + # data + two_lines = "1234\nhello world" + + all_data = (two_lines, ) + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + :note: requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + + istream = IStream(Blob.type, len(data), StringIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.sha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 40 # for now we require 40 byte shas as default + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert Blob.type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == Blob.type + assert ostream.size == len(data) + else: + self.failUnlessRaises(BadObject, db.info, sha) + self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.sha is not None + prev_sha = istream.sha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.sha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode + + @with_bare_rw_repo + def test_writing(self, rwrepo): + ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + + # write data + self._assert_object_writing(ldb) + diff --git a/test/test_stream.py b/test/test_stream.py new file mode 100644 index 000000000..020fe6bd3 --- /dev/null +++ b/test/test_stream.py @@ -0,0 +1,172 @@ +"""Test for object db""" +from test.testlib import * +from lib import ( + DummyStream, + DeriveTest, + Sha1Writer + ) + +from git.odb import * +from git import Blob +from cStringIO import StringIO +import tempfile +import os +import zlib + + + + +class TestStream(TestBase): + """Test stream classes""" + + data_sizes = (15, 10000, 1000*1024+512) + + def test_streams(self): + # test info + sha = Blob.NULL_HEX_SHA + s = 20 + info = OInfo(sha, Blob.type, s) + assert info.sha == sha + assert info.type == Blob.type + assert info.size == s + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # derive with own args + DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(Blob.type, s, stream) + assert istream.sha == None + istream.sha = sha + assert istream.sha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == Blob.type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) / ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i*ss:(i+1)*ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(Blob.type, cdata)) + type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert type == Blob.type + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + def rewind(r): + r._zip = zlib.decompressobj() + r._br = r._cws = r._cwe = 0 + if with_size: + r._parse_header_info() + # END skip header + # END make rewind func + + self._assert_stream_reader(reader, cdata, rewind) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + #zdi# + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write("hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write("hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + # its closed already + self.failUnlessRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY) + written_data = os.read(fd, os.path.getsize(path)) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 000000000..34572b37e --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,15 @@ +"""Test for object db""" +from test.testlib import * +from git import Blob +from git.odb.utils import ( + to_hex_sha, + to_bin_sha + ) + + +class TestUtils(TestBase): + def test_basics(self): + assert to_hex_sha(Blob.NULL_HEX_SHA) == Blob.NULL_HEX_SHA + assert len(to_bin_sha(Blob.NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(Blob.NULL_HEX_SHA)) == Blob.NULL_HEX_SHA + diff --git a/utils.py b/utils.py new file mode 100644 index 000000000..6863e97b9 --- /dev/null +++ b/utils.py @@ -0,0 +1,38 @@ +import binascii +import os +import errno + +#{ Routines + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +isdir = os.path.isdir +rename = os.rename +dirname = os.path.dirname +join = os.path.join +read = os.read +write = os.write +close = os.close + + +#} END Routines + + From 94c2167bb08c65d4354941e14b1899449efb04f7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 16:14:00 +0200 Subject: [PATCH 0002/3719] Adjusted imports and tests to deal with new folder structure --- db.py | 5 +- exc.py | 14 +++++ fun.py | 2 +- stream.py | 6 +-- test/lib.py | 67 +++++++++++++++++++++++- test/performance/lib.py | 49 ++++++++++++++++++ test/performance/test_db.py | 15 ++++++ test/performance/test_stream.py | 91 +++++++++++++++++++++++++++++++++ test/test_db.py | 28 +++++----- test/test_stream.py | 33 +++++++----- test/test_util.py | 15 ++++++ test/test_utils.py | 15 ------ typ.py | 10 ++++ util.py | 75 +++++++++++++++++++++++++++ utils.py | 38 -------------- 15 files changed, 376 insertions(+), 87 deletions(-) create mode 100644 exc.py create mode 100644 test/performance/lib.py create mode 100644 test/performance/test_db.py create mode 100644 test/performance/test_stream.py create mode 100644 test/test_util.py delete mode 100644 test/test_utils.py create mode 100644 typ.py create mode 100644 util.py delete mode 100644 utils.py diff --git a/db.py b/db.py index 5d3cc6a3f..7ec8a24b3 100644 --- a/db.py +++ b/db.py @@ -1,6 +1,5 @@ """Contains implementations of database retrieveing objects""" -from git.utils import IndexFileSHA1Writer -from git.errors import ( +from exc import ( InvalidDBRoot, BadObject, BadObjectType @@ -14,7 +13,7 @@ OInfo ) -from utils import ( +from util import ( ENOENT, to_hex_sha, exists, diff --git a/exc.py b/exc.py new file mode 100644 index 000000000..3eaf5777d --- /dev/null +++ b/exc.py @@ -0,0 +1,14 @@ +"""Module with common exceptions""" + +class ODBError(Exception): + """All errors thrown by the object database""" + +class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + +class BadObject(ODBError): + """The object with the given SHA does not exist""" + +class BadObjectType(ODBError): + """The object had an unsupported type""" + diff --git a/fun.py b/fun.py index 3321a8ea4..80b0f41b6 100644 --- a/fun.py +++ b/fun.py @@ -2,7 +2,7 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from git.errors import ( +from exc import ( BadObjectType ) diff --git a/stream.py b/stream.py index da97cf5b6..309df28c4 100644 --- a/stream.py +++ b/stream.py @@ -1,11 +1,11 @@ import zlib from cStringIO import StringIO -from git.utils import make_sha import errno -from utils import ( +from util import ( to_hex_sha, - to_bin_sha, + to_bin_sha, + make_sha, write, close ) diff --git a/test/lib.py b/test/lib.py index d51997488..071e38a5f 100644 --- a/test/lib.py +++ b/test/lib.py @@ -1,12 +1,75 @@ """Utilities used in ODB testing""" -from git.odb import ( +from gitdb import ( OStream, ) -from git.odb.stream import Sha1Writer +from gitdb.stream import Sha1Writer +import sys import zlib +import random +from array import array from cStringIO import StringIO +import unittest +import tempfile +import shutil +import os + + +#{ Bases + +class TestBase(unittest.TestCase): + """Base class for all tests""" + + +#} END bases + +#{ Decorators + +def with_rw_directory(func): + """Create a temporary directory which can be written to, remove it if the + test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): + path = tempfile.mktemp(suffix=func.__name__) + os.mkdir(path) + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + raise + else: + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + + +#} END decorators + +#{ Routines + +def make_bytes(size_in_bytes, randomize=False): + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tostring() + + +def make_object(type, data): + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata + data + +#} END routines + #{ Stream Utilities class DummyStream(object): diff --git a/test/performance/lib.py b/test/performance/lib.py new file mode 100644 index 000000000..03788c081 --- /dev/null +++ b/test/performance/lib.py @@ -0,0 +1,49 @@ +"""Contains library functions""" +import os +from gitdb.test.lib import * +import shutil +import tempfile + + +#{ Invvariants +k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" +#} END invariants + + +#{ Utilities +def resolve_or_fail(env_var): + """:return: resolved environment variable or raise EnvironmentError""" + try: + return os.environ[env_var] + except KeyError: + raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) + # END exception handling + +#} END utilities + + +#{ Base Classes + +class TestBigRepoR(TestBase): + """TestCase providing access to readonly 'big' repositories using the following + member variables: + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git""" + + #{ Invariants + head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' + head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' + #} END invariants + + @classmethod + def setUpAll(cls): + try: + super(TestBigRepoR, cls).setUpAll() + except AttributeError: + pass + cls.gitrepopath = resolve_or_fail(k_env_git_repo) + + +#} END base classes diff --git a/test/performance/test_db.py b/test/performance/test_db.py new file mode 100644 index 000000000..cd231b650 --- /dev/null +++ b/test/performance/test_db.py @@ -0,0 +1,15 @@ +"""Performance tests for object store""" + +import sys +from time import time + +from lib import ( + TestBigRepoR + ) + +class TestGitDBPerformance(TestBigRepoR): + + def test_random_access(self): + pass + # TODO: use the actual db for this + diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py new file mode 100644 index 000000000..2880c9b79 --- /dev/null +++ b/test/performance/test_stream.py @@ -0,0 +1,91 @@ +"""Performance data streaming performance""" + +from lib import TestBigRepoR +from gitdb.db import * +from gitdb.stream import * + +from cStringIO import StringIO +from time import time +import os +import sys +import stat +import subprocess + + +from lib import ( + TestBigRepoR, + make_bytes, + with_rw_directory + ) + + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) + + +class TestObjDBPerformance(TestBigRepoR): + + large_data_size_bytes = 1000*1000*10 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).sha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(sha) + fsize_kib = os.path.getsize(db_file) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so git has something to do + os.remove(db_file) + + # END for each randomization factor diff --git a/test/test_db.py b/test/test_db.py index 35ba86802..7f58f4f00 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,12 +1,14 @@ """Test for object db""" -from test.testlib import * -from lib import ZippedStoreShaWriter - -from git.odb import * -from git.odb.stream import Sha1Writer -from git import Blob -from git.errors import BadObject +from lib import ( + with_rw_directory, + ZippedStoreShaWriter, + TestBase + ) +from gitdb import * +from gitdb.stream import Sha1Writer +from gitdb.exc import BadObject +from gitdb.typ import str_blob_type from cStringIO import StringIO import os @@ -36,7 +38,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(Blob.type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), StringIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -48,12 +50,12 @@ def _assert_object_writing(self, db): # verify data - the slow way, we want to run code if not dry_run: info = db.info(sha) - assert Blob.type == info.type + assert str_blob_type == info.type assert info.size == len(data) ostream = db.stream(sha) assert ostream.read() == data - assert ostream.type == Blob.type + assert ostream.type == str_blob_type assert ostream.size == len(data) else: self.failUnlessRaises(BadObject, db.info, sha) @@ -81,9 +83,9 @@ def _assert_object_writing(self, db): # END for each data set # END for each dry_run mode - @with_bare_rw_repo - def test_writing(self, rwrepo): - ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + @with_rw_directory + def test_writing(self, path): + ldb = LooseObjectDB(path) # write data self._assert_object_writing(ldb) diff --git a/test/test_stream.py b/test/test_stream.py index 020fe6bd3..af7fdc35a 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -1,13 +1,22 @@ """Test for object db""" -from test.testlib import * from lib import ( + TestBase, DummyStream, DeriveTest, - Sha1Writer + Sha1Writer, + make_bytes, + make_object + ) + +from gitdb import * +from gitdb.util import ( + NULL_HEX_SHA + ) + +from gitdb.typ import ( + str_blob_type ) -from git.odb import * -from git import Blob from cStringIO import StringIO import tempfile import os @@ -23,11 +32,11 @@ class TestStream(TestBase): def test_streams(self): # test info - sha = Blob.NULL_HEX_SHA + sha = NULL_HEX_SHA s = 20 - info = OInfo(sha, Blob.type, s) + info = OInfo(sha, str_blob_type, s) assert info.sha == sha - assert info.type == Blob.type + assert info.type == str_blob_type assert info.size == s # test ostream @@ -40,10 +49,10 @@ def test_streams(self): assert stream.bytes == 20 # derive with own args - DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() # test istream - istream = IStream(Blob.type, s, stream) + istream = IStream(str_blob_type, s, stream) assert istream.sha == None istream.sha = sha assert istream.sha == sha @@ -54,7 +63,7 @@ def test_streams(self): assert istream.size == s istream.size = s * 2 istream.size == s * 2 - assert istream.type == Blob.type + assert istream.type == str_blob_type istream.type = "something" assert istream.type == "something" assert istream.stream is stream @@ -104,10 +113,10 @@ def test_decompress_reader(self): # create reader if with_size: # need object data - zdata = zlib.compress(make_object(Blob.type, cdata)) + zdata = zlib.compress(make_object(str_blob_type, cdata)) type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) - assert type == Blob.type + assert type == str_blob_type else: # here we need content data zdata = zlib.compress(cdata) diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 000000000..5aac5b84b --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,15 @@ +"""Test for object db""" +from lib import TestBase +from gitdb.util import ( + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA + ) + + +class TestUtils(TestBase): + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + diff --git a/test/test_utils.py b/test/test_utils.py deleted file mode 100644 index 34572b37e..000000000 --- a/test/test_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Test for object db""" -from test.testlib import * -from git import Blob -from git.odb.utils import ( - to_hex_sha, - to_bin_sha - ) - - -class TestUtils(TestBase): - def test_basics(self): - assert to_hex_sha(Blob.NULL_HEX_SHA) == Blob.NULL_HEX_SHA - assert len(to_bin_sha(Blob.NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(Blob.NULL_HEX_SHA)) == Blob.NULL_HEX_SHA - diff --git a/typ.py b/typ.py new file mode 100644 index 000000000..54a1f84be --- /dev/null +++ b/typ.py @@ -0,0 +1,10 @@ +"""Module containing information about types known to the database""" + +#{ String types + +str_blob_type = "blob" +str_commit_type = "commit" +str_tree_type = "tree" +str_tag_type = "tag" + +#} END string types diff --git a/util.py b/util.py new file mode 100644 index 000000000..a6f726399 --- /dev/null +++ b/util.py @@ -0,0 +1,75 @@ +import binascii +import os +import errno + +try: + import hashlib +except ImportError: + import sha + + +#{ Aliases + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +isdir = os.path.isdir +rename = os.rename +dirname = os.path.dirname +join = os.path.join +read = os.read +write = os.write +close = os.close + +# constants +NULL_HEX_SHA = "0"*40 + +#} END Aliases + + +#{ Routines + +def make_sha(source=''): + """A python2.4 workaround for the sha/hashlib module fiasco + :note: From the dulwich project """ + try: + return hashlib.sha1(source) + except NameError: + sha1 = sha.sha(source) + return sha1 + +def stream_copy(source, destination, chunk_size=512*1024): + """Copy all data from the source stream into the destination stream in chunks + of size chunk_size + + :return: amount of bytes written""" + br = 0 + while True: + chunk = source.read(chunk_size) + destination.write(chunk) + br += len(chunk) + if len(chunk) < chunk_size: + break + # END reading output stream + return br + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + + +#} END routines + diff --git a/utils.py b/utils.py deleted file mode 100644 index 6863e97b9..000000000 --- a/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -import binascii -import os -import errno - -#{ Routines - -hex_to_bin = binascii.a2b_hex -bin_to_hex = binascii.b2a_hex - -def to_hex_sha(sha): - """:return: hexified version of sha""" - if len(sha) == 40: - return sha - return bin_to_hex(sha) - -def to_bin_sha(sha): - if len(sha) == 20: - return sha - return hex_to_bin(sha) - -# errors -ENOENT = errno.ENOENT - -# os shortcuts -exists = os.path.exists -mkdir = os.mkdir -isdir = os.path.isdir -rename = os.rename -dirname = os.path.dirname -join = os.path.join -read = os.read -write = os.write -close = os.close - - -#} END Routines - - From 93f7316425128a498e0581eca12ef7b44380a1ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 17:18:34 +0200 Subject: [PATCH 0003/3719] Added async as submodule tests: minimal reorganzation of code --- .gitmodules | 3 +++ ext/async | 1 + test/lib.py | 7 ++++++- test/performance/test_stream.py | 9 +-------- 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 .gitmodules create mode 160000 ext/async diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..45ddc0b4c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/async"] + path = ext/async + url = git://gitorious.org/git-python/async.git diff --git a/ext/async b/ext/async new file mode 160000 index 000000000..5a13dc577 --- /dev/null +++ b/ext/async @@ -0,0 +1 @@ +Subproject commit 5a13dc5772ec3b00b75c8e3b533051cfb82c4929 diff --git a/test/lib.py b/test/lib.py index 071e38a5f..f0c4064ab 100644 --- a/test/lib.py +++ b/test/lib.py @@ -62,11 +62,16 @@ def make_bytes(size_in_bytes, randomize=False): a = array('i', producer) return a.tostring() - def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata + data + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) #} END routines diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 2880c9b79..8916d3e58 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -14,18 +14,11 @@ from lib import ( TestBigRepoR, - make_bytes, + make_memory_file, with_rw_directory ) -def make_memory_file(size_in_bytes, randomize=False): - """:return: tuple(size_of_stream, stream) - :param randomize: try to produce a very random stream""" - d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) - - class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it From 10fef8f8e4ee83cf54feadbb5ffb522efec739fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 17:18:51 +0200 Subject: [PATCH 0004/3719] Added project information --- AUTHORS | 1 + README | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 AUTHORS create mode 100644 README diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..490baad8e --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Creator: Sebastian Thiel diff --git a/README b/README new file mode 100644 index 000000000..a52d9f508 --- /dev/null +++ b/README @@ -0,0 +1,41 @@ +GtDB +===== + +GitDB allows you to access bare git repositories for reading and writing. It +aims at allowing full access to loose objects as well as packs with performance +and scalability in mind. It operates exclusively on streams, allowing to operate +on large objects with a small memory footprint. + +REQUIREMENTS +============ + +* Python Nose - for running the tests + +SOURCE +====== +The source is available in a git repository at gitorious and github: + +git://gitorious.org/git-python/gitdb.git +git://github.com/Byron/gitdb.git + +Once the clone is complete, please be sure to initialize the submodules using + + cd gitdb + git submodule update --init + +Run the tests with + + nosetests + +MAILING LIST +============ +http://groups.google.com/group/git-python + +ISSUE TRACKER +============= +http://byronimo.lighthouseapp.com/projects/51787-gitpython + +LICENSE +======= + +New BSD License From c64a9741a648526a3d24780ccd5ca193f48684c5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 20:36:00 +0200 Subject: [PATCH 0005/3719] Implemented all async methods, including test which shows how to chain the async method together --- __init__.py | 12 +++++++ db.py | 55 +++++++++++++++++++++--------- ext/async | 2 +- test/__init__.py | 11 ++++++ test/lib.py | 2 +- test/test_db.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++- util.py | 10 ++++++ 7 files changed, 161 insertions(+), 19 deletions(-) diff --git a/__init__.py b/__init__.py index 5789d7eb7..8b0e47b19 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,17 @@ """Initialize the object database module""" +import sys +import os + +#{ Initialization +def _init_externals(): + """Initialize external projects by putting them into the path""" + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext')) + +#} END initialization + +_init_externals() + # default imports from db import * from stream import * diff --git a/db.py b/db.py index 7ec8a24b3..8107fee25 100644 --- a/db.py +++ b/db.py @@ -14,6 +14,7 @@ ) from util import ( + pool, ENOENT, to_hex_sha, exists, @@ -32,6 +33,11 @@ stream_copy ) + +from async import ( + ChannelThreadTask + ) + import tempfile import mmap import os @@ -40,6 +46,7 @@ __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) + class ObjectDBR(object): """Defines an interface for object database lookup. Objects are identified either by hex-sha (40 bytes) or @@ -52,21 +59,30 @@ def __contains__(self, sha): def has_object(self, sha): """ :return: True if the object identified by the given 40 byte hexsha or 20 bytes - binary sha is contained in the database - :raise BadObject:""" + binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") + def has_object_async(self, reader): + """Return a reader yielding information about the membership of objects + as identified by shas + :param reader: Reader yielding 20 byte or 40 byte shas. + :return: async.Reader yielding tuples of (sha, bool) pairs which indicate + whether the given sha exists in the database or not""" + task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) + return pool.add_task(task) + def info(self, sha): """ :return: OInfo instance :param sha: 40 bytes hexsha or 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def info_async(self, input_channel): + def info_async(self, reader): """Retrieve information of a multitude of objects asynchronously - :param input_channel: Channel yielding the sha's of the objects of interest - :return: Channel yielding OInfo|InvalidOInfo, in any order""" - raise NotImplementedError("To be implemented in subclass") + :param reader: Channel yielding the sha's of the objects of interest + :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" + task = ChannelThreadTask(reader, str(self.info_async), self.info) + return pool.add_task(task) def stream(self, sha): """:return: OStream instance @@ -74,12 +90,17 @@ def stream(self, sha): :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def stream_async(self, input_channel): + def stream_async(self, reader): """Retrieve the OStream of multiple objects - :param input_channel: see ``info`` + :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` - :return: Channel yielding OStream|InvalidOStream instances in any order""" - raise NotImplementedError("To be implemented in subclass") + :return: async.Reader yielding OStream|InvalidOStream instances in any order + :note: depending on the system configuration, it might not be possible to + read all OStreams at once. Instead, read them individually using reader.read(x) + where x is small enough.""" + # base implementation just uses the stream method repeatedly + task = ChannelThreadTask(reader, str(self.stream_async), self.stream) + return pool.add_task(task) #} END query interface @@ -114,7 +135,7 @@ def store(self, istream): :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - def store_async(self, input_channel): + def store_async(self, reader): """Create multiple new objects in the database asynchronously. The method will return right away, returning an output channel which receives the results as they are computed. @@ -122,13 +143,15 @@ def store_async(self, input_channel): :return: Channel yielding your IStream which served as input, in any order. The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. - :param input_channel: Channel yielding IStream instance. - As the same instances will be used in the output channel, you can create a map - between the id(istream) -> istream - :note:As some ODB implementations implement this operation as atomic, they might + :param reader: async.Reader yielding IStream instances. + The same instances will be used in the output channel as were received + in by the Reader. + :note:As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" - raise NotImplementedError("To be implemented in subclass") + # base implementation uses store to perform the work + task = ChannelThreadTask(reader, str(self.store_async), self.store) + return pool.add_task(task) #} END edit interface diff --git a/ext/async b/ext/async index 5a13dc577..164bb702e 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 5a13dc5772ec3b00b75c8e3b533051cfb82c4929 +Subproject commit 164bb702e3871ab30341a714ef517fb58cd76772 diff --git a/test/__init__.py b/test/__init__.py index 8b1378917..0dec7750f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1 +1,12 @@ +import gitdb.util + +#{ Initialization +def _init_pool(): + """Assure the pool is actually threaded""" + size = 2 + print "Setting ThreadPool to %i" % size + gitdb.util.pool.set_size(size) + + +#} END initialization diff --git a/test/lib.py b/test/lib.py index f0c4064ab..fc0982bcb 100644 --- a/test/lib.py +++ b/test/lib.py @@ -30,7 +30,7 @@ def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" def wrapper(self): - path = tempfile.mktemp(suffix=func.__name__) + path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) try: return func(self, path) diff --git a/test/test_db.py b/test/test_db.py index 7f58f4f00..7ba770f48 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -10,6 +10,8 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type +from async import IteratorReader + from cStringIO import StringIO import os @@ -78,10 +80,93 @@ def _assert_object_writing(self, db): new_ostream = db.ostream() # note: only works as long our store write uses the same compression - # level, which is zip + # level, which is zip_best assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode + + def _assert_object_writing_async(self, db): + """Test generic object writing using asynchronous access""" + ni = 5000 + def istream_generator(offset=0, ni=ni): + for data_src in xrange(ni): + data = str(data_src + offset) + yield IStream(str_blob_type, len(data), StringIO(data)) + # END for each item + # END generator utility + + # for now, we are very trusty here as we expect it to work if it worked + # in the single-stream case + + # write objects + reader = IteratorReader(istream_generator()) + istream_reader = db.store_async(reader) + istreams = istream_reader.read() # read all + assert istream_reader.task().error() is None + assert len(istreams) == ni + + for stream in istreams: + assert stream.error is None + assert len(stream.sha) == 40 + assert isinstance(stream, IStream) + # END assert each stream + + # test has-object-async - we must have all previously added ones + reader = IteratorReader( istream.sha for istream in istreams ) + hasobject_reader = db.has_object_async(reader) + count = 0 + for sha, has_object in hasobject_reader: + assert has_object + count += 1 + # END for each sha + assert count == ni + + # read the objects we have just written + reader = IteratorReader( istream.sha for istream in istreams ) + ostream_reader = db.stream_async(reader) + + # read items individually to prevent hitting possible sys-limits + count = 0 + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert ostream_reader.task().error() is None + assert count == ni + + # get info about our items + reader = IteratorReader( istream.sha for istream in istreams ) + info_reader = db.info_async(reader) + + count = 0 + for oinfo in info_reader: + assert isinstance(oinfo, OInfo) + count += 1 + # END for each oinfo instance + assert count == ni + + + # combined read-write using a converter + # add 2500 items, and obtain their output streams + nni = 2500 + reader = IteratorReader(istream_generator(offset=ni, ni=nni)) + istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ] + + istream_reader = db.store_async(reader) + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = db.stream_async(istream_reader) + + count = 0 + # read it individually, otherwise we might run into the ulimit + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert count == nni + + + @with_rw_directory def test_writing(self, path): @@ -89,4 +174,5 @@ def test_writing(self, path): # write data self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) diff --git a/util.py b/util.py index a6f726399..fd52695bf 100644 --- a/util.py +++ b/util.py @@ -2,11 +2,21 @@ import os import errno +from async import ThreadPool + try: import hashlib except ImportError: import sha +#{ Globals + +# A pool distributing tasks, initially with zero threads, hence everything +# will be handled in the main thread +pool = ThreadPool(0) + +#} END globals + #{ Aliases From 05cee2eb6b35d5216d4dd34bed50cdc921668cd4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 22:22:33 +0200 Subject: [PATCH 0006/3719] Added multi-threading performance tests which show that, during compression and decompression, it is not a tiny bit faster than without, which is due to the GIL and even separately locked zlib module implementations. The only way to make this faster update the resepctive c modules to drop the gil, and their own locks where possible --- ext/async | 2 +- test/performance/test_stream.py | 102 +++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/ext/async b/ext/async index 164bb702e..8cfa2542e 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 164bb702e3871ab30341a714ef517fb58cd76772 +Subproject commit 8cfa2542ed623627b5e2e91072368209710e9370 diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 8916d3e58..298207963 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -3,6 +3,14 @@ from lib import TestBigRepoR from gitdb.db import * from gitdb.stream import * +from gitdb.util import pool +from gitdb.typ import str_blob_type +from gitdb.fun import chunk_size + +from async import ( + IteratorReader, + ChannelThreadTask, + ) from cStringIO import StringIO from time import time @@ -19,6 +27,30 @@ ) +#{ Utilities +def read_chunked_stream(stream): + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + +class TestStreamReader(ChannelThreadTask): + """Expects input streams and reads them in chunks. It will read one at a time, + requireing a queue chunk of size 1""" + def __init__(self, *args): + super(TestStreamReader, self).__init__(*args) + self.fun = read_chunked_stream + self.max_chunksize = 1 + + +#} END utilities + class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it @@ -27,7 +59,9 @@ class TestObjDBPerformance(TestBigRepoR): @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' print >> sys.stderr, "Creating %s data ..." % desc @@ -35,6 +69,7 @@ def test_large_data_streaming(self, path): size, stream = make_memory_file(self.large_data_size_bytes, randomize) elapsed = time() - st print >> sys.stderr, "Done (in %f s)" % elapsed + string_ios.append(stream) # writing - due to the compression it will seem faster than it is st = time() @@ -78,7 +113,70 @@ def test_large_data_streaming(self, path): cs_kib = cs / 1000 print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) - # del db file so git has something to do + # del db file so we keep something to do os.remove(db_file) - # END for each randomization factor + + + # multi-threaded mode + # want two, should be supported by most of todays cpus + pool.set_size(2) + total_kib = 0 + nsios = len(string_ios) + for stream in string_ios: + stream.seek(0) + total_kib += len(stream.getvalue()) / 1000 + # END rewind + + def istream_iter(): + for stream in string_ios: + stream.seek(0) + yield IStream(str_blob_type, len(stream.getvalue()), stream) + # END for each stream + # END util + + # write multiple objects at once, involving concurrent compression + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + st = time() + istreams = istream_reader.read(nsios) + assert len(istreams) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + + # decompress multiple at once, by reading them + istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # store the files, and read them back. For the reading, we use a task + # as well which is chunked into one item per task. Reading all will + # very quickly result in two threads handling two bytestreams of + # chained compression/decompression streams + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + + istream_to_sha = lambda items: [ i.sha for i in items ] + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = ldb.stream_async(istream_reader) + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) From 0ef86550179b9bb9e29ecccdccd586713b9d1752 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 13 Jun 2010 13:44:10 +0200 Subject: [PATCH 0007/3719] Now using the async zlib module if it is available to allow performance gains through multi-threading. --- ext/async | 2 +- fun.py | 2 +- stream.py | 5 +++-- test/lib.py | 2 +- test/performance/test_stream.py | 13 +++++++++---- test/test_stream.py | 2 +- util.py | 6 ++++++ 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ext/async b/ext/async index 8cfa2542e..77bf7bef7 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 8cfa2542ed623627b5e2e91072368209710e9370 +Subproject commit 77bf7bef748b019a3a59693cef6d955f74b358ad diff --git a/fun.py b/fun.py index 80b0f41b6..c766f8e09 100644 --- a/fun.py +++ b/fun.py @@ -6,7 +6,7 @@ BadObjectType ) -import zlib +from util import zlib decompressobj = zlib.decompressobj diff --git a/stream.py b/stream.py index 309df28c4..10bc8901a 100644 --- a/stream.py +++ b/stream.py @@ -1,4 +1,4 @@ -import zlib + from cStringIO import StringIO import errno @@ -7,7 +7,8 @@ to_bin_sha, make_sha, write, - close + close, + zlib ) __all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', diff --git a/test/lib.py b/test/lib.py index fc0982bcb..723958ca1 100644 --- a/test/lib.py +++ b/test/lib.py @@ -3,9 +3,9 @@ OStream, ) from gitdb.stream import Sha1Writer +from gitdb.util import zlib import sys -import zlib import random from array import array from cStringIO import StringIO diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 298207963..5de463ee2 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -1,5 +1,4 @@ """Performance data streaming performance""" - from lib import TestBigRepoR from gitdb.db import * from gitdb.stream import * @@ -53,7 +52,7 @@ def __init__(self, *args): class TestObjDBPerformance(TestBigRepoR): - large_data_size_bytes = 1000*1000*10 # some MiB should do it + large_data_size_bytes = 1000*1000*50 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB @with_rw_directory @@ -147,19 +146,22 @@ def istream_iter(): print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - # decompress multiple at once, by reading them + # chunk size is not important as the stream will not really be decompressed + + # until its read istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) ostream_reader = ldb.stream_async(istream_reader) chunk_task = TestStreamReader(ostream_reader, "chunker", None) output_reader = pool.add_task(chunk_task) + output_reader.task().max_chunksize = 1 st = time() assert len(output_reader.read(nsios)) == nsios elapsed = time() - st - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) # store the files, and read them back. For the reading, we use a task # as well which is chunked into one item per task. Reading all will @@ -167,13 +169,16 @@ def istream_iter(): # chained compression/decompression streams reader = IteratorReader(istream_iter()) istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 istream_to_sha = lambda items: [ i.sha for i in items ] istream_reader.set_post_cb(istream_to_sha) ostream_reader = ldb.stream_async(istream_reader) + chunk_task = TestStreamReader(ostream_reader, "chunker", None) output_reader = pool.add_task(chunk_task) + output_reader.max_chunksize = 1 st = time() assert len(output_reader.read(nsios)) == nsios diff --git a/test/test_stream.py b/test/test_stream.py index af7fdc35a..4f022286e 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -13,6 +13,7 @@ NULL_HEX_SHA ) +from gitdb.util import zlib from gitdb.typ import ( str_blob_type ) @@ -20,7 +21,6 @@ from cStringIO import StringIO import tempfile import os -import zlib diff --git a/util.py b/util.py index fd52695bf..6b8862472 100644 --- a/util.py +++ b/util.py @@ -2,6 +2,12 @@ import os import errno +try: + import async.mod.zlib as zlib +except ImportError: + import zlib +# END try async zlib + from async import ThreadPool try: From 97a17dc4f3af188c8c00b0b265f17c26c9c96ddc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jun 2010 18:16:31 +0200 Subject: [PATCH 0008/3719] Made the db module a package to have enough room for expansion --- db/__init__.py | 7 ++ db.py => db/base.py | 218 +------------------------------------------- db/git.py | 33 +++++++ db/loose.py | 186 +++++++++++++++++++++++++++++++++++++ db/pack.py | 11 +++ db/ref.py | 7 ++ ext/async | 2 +- 7 files changed, 250 insertions(+), 214 deletions(-) create mode 100644 db/__init__.py rename db.py => db/base.py (50%) create mode 100644 db/git.py create mode 100644 db/loose.py create mode 100644 db/pack.py create mode 100644 db/ref.py diff --git a/db/__init__.py b/db/__init__.py new file mode 100644 index 000000000..05d9b21b3 --- /dev/null +++ b/db/__init__.py @@ -0,0 +1,7 @@ + +from base import * +from loose import * +from pack import * +from git import * +from ref import * + diff --git a/db.py b/db/base.py similarity index 50% rename from db.py rename to db/base.py index 8107fee25..2cda0ea0a 100644 --- a/db.py +++ b/db/base.py @@ -1,50 +1,15 @@ """Contains implementations of database retrieveing objects""" -from exc import ( - InvalidDBRoot, - BadObject, - BadObjectType - ) - -from stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - Sha1Writer, - OStream, - OInfo - ) - -from util import ( +from gitdb.util import ( pool, - ENOENT, - to_hex_sha, - exists, - hex_to_bin, - isdir, - mkdir, - rename, - dirname, join ) -from fun import ( - chunk_size, - loose_object_header_info, - write_object, - stream_copy - ) - - from async import ( ChannelThreadTask ) -import tempfile -import mmap -import os - -__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', - 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB') class ObjectDBR(object): @@ -104,6 +69,7 @@ def stream_async(self, reader): #} END query interface + class ObjectDBW(object): """Defines an interface to create objects in the database""" @@ -183,181 +149,7 @@ def db_path(self, rela_path): return join(self._root_path, rela_path) #} END interface - - -class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): - """A database which operates on loose object files""" - - # CONFIGURATION - # chunks in which data will be copied between streams - stream_chunk_size = chunk_size - - - def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) - self._hexsha_to_file = dict() - # Additional Flags - might be set to 0 after the first failure - # Depending on the root, this might work for some mounts, for others not, which - # is why it is per instance - self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface - def object_path(self, hexsha): - """ - :return: path at which the object with the given hexsha would be stored, - relative to the database root""" - return join(hexsha[:2], hexsha[2:]) - - def readable_db_object_path(self, hexsha): - """ - :return: readable object path to the object identified by hexsha - :raise BadObject: If the object file does not exist""" - try: - return self._hexsha_to_file[hexsha] - except KeyError: - pass - # END ignore cache misses - - # try filesystem - path = self.db_path(self.object_path(hexsha)) - if exists(path): - self._hexsha_to_file[hexsha] = path - return path - # END handle cache - raise BadObject(hexsha) - - #} END interface - - def _map_loose_object(self, sha): - """ - :return: memory map of that file to allow random read access - :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(to_hex_sha(sha))) - try: - fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) - except OSError,e: - if e.errno != ENOENT: - # try again without noatime - try: - fd = os.open(db_path, os.O_RDONLY) - except OSError: - raise BadObject(to_hex_sha(sha)) - # didn't work because of our flag, don't try it again - self._fd_open_flags = 0 - else: - raise BadObject(to_hex_sha(sha)) - # END handle error - # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed - - def set_ostream(self, stream): - """:raise TypeError: if the stream does not support the Sha1Writer interface""" - if stream is not None and not isinstance(stream, Sha1Writer): - raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) - - def info(self, sha): - m = self._map_loose_object(sha) - try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) - finally: - m.close() - # END assure release of system resources - - def stream(self, sha): - m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) - return OStream(sha, type, size, stream) - - def has_object(self, sha): - try: - self.readable_db_object_path(to_hex_sha(sha)) - return True - except BadObject: - return False - # END check existance - - def store(self, istream): - """note: The sha we produce will be hex by nature""" - tmp_path = None - writer = self.ostream() - if writer is None: - # open a tmp file to write the data to - fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - writer = FDCompressedSha1Writer(fd) - # END handle custom writer - - try: - try: - if istream.sha is not None: - stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) - else: - # write object with header, we have to make a new one - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) - # END handle direct stream copies - except: - if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - finally: - if tmp_path: - writer.close() - # END assure target stream is closed - - sha = istream.sha or writer.sha(as_hex=True) - - if tmp_path: - obj_path = self.db_path(self.object_path(sha)) - obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) - # END handle destination directory - rename(tmp_path, obj_path) - # END handle dry_run - - istream.sha = sha - return istream - - -class PackedDB(FileDBBase, ObjectDBR): - """A database operating on a set of object packs""" - - + class CompoundDB(ObjectDBR): """A database which delegates calls to sub-databases""" - - -class ReferenceDB(CompoundDB): - """A database consisting of database referred to in a file""" - - -#class GitObjectDB(CompoundDB, ObjectDBW): -class GitObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file - - It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations - """ - def __init__(self, root_path, git): - """Initialize this instance with the root and a git command""" - super(GitObjectDB, self).__init__(root_path) - self._git = git - - def info(self, sha): - t = self._git.get_object_header(sha) - return OInfo(*t) - - def stream(self, sha): - """For now, all lookup is done by git itself""" - t = self._git.stream_object_data(sha) - return OStream(*t) - + # TODO diff --git a/db/git.py b/db/git.py new file mode 100644 index 000000000..d2477d7b1 --- /dev/null +++ b/db/git.py @@ -0,0 +1,33 @@ + +from gitdb.stream import ( + OInfo, + OStream + ) + +from loose import LooseObjectDB + +__all__ = ('GitObjectDB', ) + +#class GitObjectDB(CompoundDB, ObjectDBW): +class GitObjectDB(LooseObjectDB): + """A database representing the default git object store, which includes loose + objects, pack files and an alternates file + + It will create objects only in the loose object database. + :note: for now, we use the git command to do all the lookup, just until he + have packs and the other implementations + """ + def __init__(self, root_path, git): + """Initialize this instance with the root and a git command""" + super(GitObjectDB, self).__init__(root_path) + self._git = git + + def info(self, sha): + t = self._git.get_object_header(sha) + return OInfo(*t) + + def stream(self, sha): + """For now, all lookup is done by git itself""" + t = self._git.stream_object_data(sha) + return OStream(*t) + diff --git a/db/loose.py b/db/loose.py new file mode 100644 index 000000000..37aad8c6f --- /dev/null +++ b/db/loose.py @@ -0,0 +1,186 @@ +from base import ( + FileDBBase, + ObjectDBR, + ObjectDBW + ) + + +from gitdb.exc import ( + InvalidDBRoot, + BadObject, + ) + +from gitdb.stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + Sha1Writer, + OStream, + OInfo + ) + +from gitdb.util import ( + ENOENT, + to_hex_sha, + exists, + isdir, + mkdir, + rename, + dirname, + join + ) + +from gitdb.fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) + +import tempfile +import mmap +import os + + +__all__ = ( 'LooseObjectDB', ) + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(to_hex_sha(sha))) + try: + fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + raise BadObject(to_hex_sha(sha)) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(to_hex_sha(sha)) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(to_hex_sha(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + writer = FDCompressedSha1Writer(fd) + # END handle custom writer + + try: + try: + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + + sha = istream.sha or writer.sha(as_hex=True) + + if tmp_path: + obj_path = self.db_path(self.object_path(sha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + rename(tmp_path, obj_path) + # END handle dry_run + + istream.sha = sha + return istream + diff --git a/db/pack.py b/db/pack.py new file mode 100644 index 000000000..e57241a32 --- /dev/null +++ b/db/pack.py @@ -0,0 +1,11 @@ +"""Module containing a database to deal with packs""" +from base import ( + FileDBBase, + ObjectDBR + ) + +__all__ = ('PackedDB', ) + +class PackedDB(FileDBBase, ObjectDBR): + """A database operating on a set of object packs""" + diff --git a/db/ref.py b/db/ref.py new file mode 100644 index 000000000..2c63884bc --- /dev/null +++ b/db/ref.py @@ -0,0 +1,7 @@ +from base import CompoundDB + +__all__ = ('CompoundDB', ) + +class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" + diff --git a/ext/async b/ext/async index 77bf7bef7..af0040b0f 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 77bf7bef748b019a3a59693cef6d955f74b358ad +Subproject commit af0040b0f3c6ede3be5b2d6bc69f6ea5ac53c36c From 133988a9b53400810d2baea9cc817c67dd1577a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jun 2010 18:27:29 +0200 Subject: [PATCH 0009/3719] update db test to allow testing of individual database types, giving more fine-grained control over testing the db packge --- test/db/__init__.py | 0 test/{test_db.py => db/lib.py} | 31 ++++++++++++++----------------- test/db/test_loose.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 17 deletions(-) create mode 100644 test/db/__init__.py rename test/{test_db.py => db/lib.py} (93%) create mode 100644 test/db/test_loose.py diff --git a/test/db/__init__.py b/test/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_db.py b/test/db/lib.py similarity index 93% rename from test/test_db.py rename to test/db/lib.py index 7ba770f48..8d61e677b 100644 --- a/test/test_db.py +++ b/test/db/lib.py @@ -1,22 +1,29 @@ -"""Test for object db""" -from lib import ( +"""Base classes for object db testing""" +from gitdb.test.lib import ( with_rw_directory, ZippedStoreShaWriter, TestBase ) -from gitdb import * -from gitdb.stream import Sha1Writer +from gitdb.stream import ( + Sha1Writer, + IStream, + OStream, + OInfo + ) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type from async import IteratorReader from cStringIO import StringIO -import os + + +__all__ = ('TestDBBase', 'with_rw_directory' ) -class TestDB(TestBase): - """Test the different db class implementations""" +class TestDBBase(TestBase): + """Base class providing testing routines on databases""" # data two_lines = "1234\nhello world" @@ -166,13 +173,3 @@ def istream_generator(offset=0, ni=ni): assert count == nni - - - @with_rw_directory - def test_writing(self, path): - ldb = LooseObjectDB(path) - - # write data - self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) - diff --git a/test/db/test_loose.py b/test/db/test_loose.py new file mode 100644 index 000000000..70cd7742c --- /dev/null +++ b/test/db/test_loose.py @@ -0,0 +1,13 @@ +from lib import * +from gitdb.db import LooseObjectDB + +class TestLooseDB(TestDBBase): + + @with_rw_directory + def test_writing(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) + From 937d592ad08cff4bcb675a96d62534c65cc8015c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 01:06:01 +0200 Subject: [PATCH 0010/3719] Added LockedFD class including test, it moved 'down' from git-python --- test/test_util.py | 78 ++++++++++++++++++++++++- util.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index 5aac5b84b..2272b53e5 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,9 +1,13 @@ """Test for object db""" +import tempfile +import os + from lib import TestBase from gitdb.util import ( to_hex_sha, to_bin_sha, - NULL_HEX_SHA + NULL_HEX_SHA, + LockedFD ) @@ -12,4 +16,76 @@ def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + fp = open(file_path, "rb") + try: + assert fp.read() == data + finally: + fp.close() + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + my_file_fp = open(my_file, "wb") + my_file_fp.write(orig_data) + my_file_fp.close() + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.failUnlessRaises(AssertionError, lfd.rollback) + self.failUnlessRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesnt fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.failUnlessRaises(IOError, olfd.open) + + wfdstream.write(new_data) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup diff --git a/util.py b/util.py index 6b8862472..291855630 100644 --- a/util.py +++ b/util.py @@ -1,5 +1,6 @@ import binascii import os +import sys import errno try: @@ -89,3 +90,146 @@ def to_bin_sha(sha): #} END routines + +#{ Utilities + + +class FDStreamWrapper(object): + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + +class LockedFD(object): + """This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + :note: with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """Open the file descriptor for reading or writing, both in binary mode. + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + :note: must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exlusive here, as we obtained the lock anyway + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + # END open descriptor for reading + + if stream: + return FDStreamWrapper(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + :note: can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + :note: can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if os.path.isfile(self._filepath): + os.remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + else: + # just delete the file so far, we failed + os.remove(lockfile) + # END successful handling + +#} END utilities From f50643ff166180d3a048ff55422d84e631e831b1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 01:07:32 +0200 Subject: [PATCH 0011/3719] Added basic frame for packfile implementation and testing, as well as the testing of the corresponding PackedDB. It wants to be filled out now, but the design not yet done either actually --- db/pack.py | 33 ++++++++++++++++ exc.py | 2 + pack.py | 1 + test/db/lib.py | 4 +- test/db/test_pack.py | 12 ++++++ ...fdfa9e156ab73caae3b6da867192221f2089c2.idx | Bin 0 -> 1912 bytes ...dfa9e156ab73caae3b6da867192221f2089c2.pack | Bin 0 -> 51875 bytes test/lib.py | 37 ++++++++++++++++++ test/test_pack.py | 16 ++++++++ 9 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 pack.py create mode 100644 test/db/test_pack.py create mode 100644 test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx create mode 100644 test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack create mode 100644 test/test_pack.py diff --git a/db/pack.py b/db/pack.py index e57241a32..a850e0fb2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -4,8 +4,41 @@ ObjectDBR ) +from gitdb.exc import ( + UnsupportedOperation, + ) + __all__ = ('PackedDB', ) class PackedDB(FileDBBase, ObjectDBR): """A database operating on a set of object packs""" + def __init__(self, root_path): + super(PackedDB, self).__init__(root_path) + + + #{ Object DB Read + + def has_object(self, sha): + raise NotImplementedError() + + def info(self, sha): + raise NotImplementedError() + + def stream(self, sha): + raise NotImplementedError() + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + def store_async(self, reader): + raise NotImplementedError() + + #} END object db write diff --git a/exc.py b/exc.py index 3eaf5777d..482726e3b 100644 --- a/exc.py +++ b/exc.py @@ -12,3 +12,5 @@ class BadObject(ODBError): class BadObjectType(ODBError): """The object had an unsupported type""" +class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/pack.py b/pack.py new file mode 100644 index 000000000..676fa26c5 --- /dev/null +++ b/pack.py @@ -0,0 +1 @@ +"""Contains PackIndex and PackFile implementations""" diff --git a/test/db/lib.py b/test/db/lib.py index 8d61e677b..738eeb851 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -1,6 +1,7 @@ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, + with_packs, ZippedStoreShaWriter, TestBase ) @@ -16,11 +17,10 @@ from gitdb.typ import str_blob_type from async import IteratorReader - from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' ) class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_pack.py b/test/db/test_pack.py new file mode 100644 index 000000000..29b348876 --- /dev/null +++ b/test/db/test_pack.py @@ -0,0 +1,12 @@ +from lib import * +from gitdb.db import PackedDB + +class TestPackDB(TestDBBase): + + @with_rw_directory + @with_packs + def test_writing(self, path): + ldb = PackedDB(path) + # TODO + + diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx new file mode 100644 index 0000000000000000000000000000000000000000..fda5969bc69c10781d3159ecfc6a3caf36d3eea9 GIT binary patch literal 1912 zcmexg;-AdGz`z8=t%7h5R~Q zTXcX;mTl)hi_XvCD|Od)tJiK`87aAUf}`>E=V$DUY<600OIVw9x2C82n)#~P2M@Ja zwOIbMP34ktToN=Rxv~G6@XomAwC?=}9jBkO$(>>z`H-WFX~s6!!YzA>qI^uhIo^J| z?vlaQi2geQIj$C4F3k3sz99UGVC2uTnjZyzSJz2M-S_Pckk%}5U!fXnYJ9AH+OGu-(2 zw2i8TzvWXCk`_i)Jiqg$`DTLNnLou=p&UZzJav6maeY%g(W3UP=ufKhMAMBQBe=KE zJ1zE?^+QYJ?e~j*PoA-?_@BJT9S&UL4_P#nPuL6x5;q0p)nH(}Y$K7@A^B4QY zKFoO;LE2$LFBol?K3 zi)vlv$=yb0{+D?ki{h$wQz{2t5R^_?jnqvlONmddNX(5LTPZTs!na$V15Z&z(BqE8Ww(Zh%a!T)|W_!#O7_n1U3oD_JRosc_f1Thc=bDcsLm_cg&T4phZO^m^jPta0JV+A8E zvs$ok?@62+r$yBZ-aJZ?nkyUQ86c_*wSGfbtxfvQ6Xw1C~%USqch}(FGMK33|-WamcqvZ$n$kNsI{a7G*SUdu%VrNl}1Hc#tcwRkVbb{?_%)vi{yF2qOM zdf{c%x}@)?Kdg!$Qg35^ml1fJos3IP!!Qs)_ddm3Akp~SF+x=(7Ercufw4VPvPfJ7 zSAyHOM?h?!-g{#g0SaVW5oM<(`%Ih;Ud3vlRs&(R8Y59gbLCyO0jxn&!02*nQd7>} z*~&4uK#{yEqj$cb7`~0KzQg-%TRObJ8?U@|c)l#-d&qve_R@dm+Semk7rpg_m|(&K zqIuAD84v4UwRt^%f}PVG=3vana@nL!U3DtIPk$JiJ87m=5}F!#oSlrzN&_(%h1cdO ze!3|LQ*)Wziil7U5qyYT{%O;8k|C3UKE8wKLKiOF9nSZi(<&kuBaJm88s;hDpcQ&4 zBWI3AtGxD1nOa7!%g_p@ zm5A%#I@J-wfD*0ZQBdVwAZ}@=QsHgMslWkV-RN?G$75=9!}>fGUv9fl#wTE03MRB< z20E%KC2KpTwp#yMlXt8~1YcbaC)l3D!{}0O-I6C4E?x0w3~lb_R#RwOSW$sbgOGdp z+ujp^mvn>8PYT@^-w<*rdAP1|-jh4Oq-#t0*VitP54%c*{gNJdoSo6nO2jY_2Jm~H zVs5KK*ZkSE%_1TQB7)B#lVq|9v`J~RY#-lxkOe&mo@V&?e#~L&Oa#<(ZjFy79QcdhijCMQ}%?^UqI2&>R{s9L=x^xaohQ{W^; z>m|p7@#v&fWKFNP#pzZH=`A^i=NiilZ|9r}Ji{xFm1J^KV*U!97T8F-wXjloI-F%(7j`HDw31!3BkWRgjXh~P#9KjI~M>5O%fl4d~v z-a+lwh1+}2;k3#OCbHs4)~KWo5_P~REhZPx#DSgm)@YJu5ErVP8+cF34Mt0^$@L}? zp;qJ)C(SkvtL;p12Y=&3Fb*j>XK)CNiUUc~Yfmcd`xxaRxIQ`*w6w8Qcv_yK?Gd=+0p zp?x!v8+e?ZjLk{|F%(Akd5XKM3Zwa-q=*P^MDQ6TxpSuz+ewBr#rpORMHj9Fx8FJ6 zX_W{_DveS}BpzKr8r7)P7}k+1OV$}Fqp?D=B~;1{wD3KWLOXb^dgDoO=K`1BMXOCO zqK?>06Mp<*Wf%;1(P>FmU@rzQxe|OF6_1g)RL*DdW5$q=(5KD|Nk z1-uo-S5#4>Gea;rE>bhmO5PZBP6nlkZPeKkqQrI4j)FL_a~+`NYHXA`D`!^|B_~AB zoc(GwW@VL?DIldk-m2({4J|oZ<6UGWLqMrxNQ-q|C7Z}=hXQY7YaL#1wKRB!7u>Nn zc(|7FF}Ob+Ew!&Rwfzy4b~-L|gok_Sr4-ZM%b3)^(&CN#l_7Pkw83e|QpZV%JNQ)> z0kDm6oaqAi*-D0W-d5@ru)=JsWfXib-<;uSljkme0FabvD47{}oSl%tN(3{aj?RW98hyMsGaDr~(tK7+v%0citJqs-im09PW3bK%ytcJec)sRT;1Qm1!(8C*oZ4IC`mh&YK8{qjdl=QMoSB@I zFi0z<=<1eQr~Z`|FI>+IzG9{o4wY^qA+5I67_ ze-Z%S){#zt?5~KOrT1_ZU#TI22%jBzoSjh1PQx$|y!$Iw zJ;9-s^K4QO0&ziH`GU2*PF5s#v0hrxFK$<(d@h=5kb)zS9x1%qnozzeQhjh ztSBEXc9khx({*jRR|_Tz0!?o^S9QLn+84d)E893**L{ViDE7KV>k3_%M&T-BOz^x0 zj_?RiIAe%#JFpyu()|)meBaqP-$79}W#86a-@sPywN@FOSu*Qi-)a!|MWfQ-Z|^< z{LXz8_wH3wQ4v#cN0Z@tm0pZcE{of zNZ_cu9H)Tl3HE+nb)_a_voEL=?LRc-1xI7N;CLXx(cR|99ea0Owx!kr;LG#-Qa%|} zbJ)!M)G^be0Pc6t5!XbK!wD|dS8GNNJa-cJ&s5n9{t?r|A;?}xEcj+<92U?&YZL9# zezA(=zSAQ08WU{zx8p&us(q@eVn>gigjwlZoH%A?3OlNA3bb|HO+!q-?!0fG#1C9b z(jWY_FPZw(W2Prg>KGdXrRK)*pMU;0_RvDxwIw-|{gd@48Zm);BHhp!B+wFnDO6TQ z2gXr}#3ue6%3!-{LSE@@+$SdX7a-52N9>y zKe;sI>VrlgL6d7Fa<*q=Goq6lEu03}U%a^B^?J2I0mD5p%$U7f-1Ee@6tj@R*ab*? z6ley6h7-YR_Q?f@#4`Q)GOVVbs%_so;S$(#cZFbZ2#{c>$(lm!v+t8DRWIn6*GLw1 zM+(sh0f9pMAYmjp0fcXRF<;(PoV~I^*++Sd?F1r9cYiel3_#;S*tpwV?~An;lXT@y z`#dR?!>!~kpH@Q<5uXj+@+xrde6c5Awr{dbCMfokI|D)iERmHbT3Fqg<5<27X2kBF z+0Vx&Mnm8T2CK1r@w8!d|+Ms}Qzvs7dXUW@N$x-I*xOZ9ghurl&G=sspfVI2`W{cU@zi8Z>j{E46 zQ9)AL7bi?FP{4>o>di}tQhB^VQ)<7~==w+g_94qt1Oy0A7!d_m1gw>wRbCp%bojQ; zJM8k^YE~Yrft|E!2yiR{8fL6NwCryTd@i&-T@ad{uz0hz|8VeXg95&{Z^<_H5i1)w zqv5Dg@O|R9tzkaRkkJ?t8ViHFx`LIx!Jb+_=QD9+^G^Abh1UJKoPDbPj1Y^(fzuc2 z-1)=bn>{VRUUR>8j3bJjd3NxM_zgyl$7B3KTC-1+<$;(&K<|HWq`>y>g`_cGFz zLBp#mlJcSI?82w$M?pXaPk{@w`3nZT+!rty3W>oZp%G#yj=280l0)Kl;rm^gla7li zr9JwLWiW6rsCqK5?*VOI%R!Hv52aWuextZg7hMLiWm+a>}54!Jj zHdkh}WUMO&syEq2KzLo zC2n-9XiKIfqedaYkwBdSm);!)5@qI$3OL`-?0vJ85w!1Wm;bc4C!2wre#IR8-> zVH+9CCJu{COKqytmaZ;l_FJnIj!c4~v93rzu*~gS-?@LmEM8kWU6}YZUG1)W@?zKj zDFfwT5qlk zCB-ckYi~b;O1*DM@`=`J+SR?5r?#X2+fG82N!_6yZLYAdOm|fk3-#F6;GkS;cxPG1 zk-WF`ADR`WOc9%BI{ z34wyaNH`n@ETH*fCgUQW`Da^pz>a*o=GU-*6vjnkxd0xH@C3a}g6h;!iOiA%lD}(3 zl3aG>?rzgv%R>vDL3L2(famGTq(t6J@^h?Fup9wK>P3QiAqnnCm=_Tm2`LuZKw+`T z4YJ$V2eK!sZTB}dImRM50t^WUUkIFoV2N9IwD{2n;&OiFa+L_1b0~Pq^_wu3AX47E z`qC!l(=94XMVxc?AtzZCI}8my7^FM2iw7UN@jA_vJ#%zlRH0Fz_u`hs?eUC`fP{8u zpDowh-aTwpcyl)ezyCtKo7ugi5)2YrJ3;w*^tRuvT~olLY+Fwrg(H{OW&UPdw$NB` zqPapFTi5xzZzZy9C3iV5ND2g--&wHzBFKO-IH}ij%LUqR4??XT$z!J;*?R$@FU&AKunDtcaOE=jiHO!2tJYTC2NS{2@4SVcycuUU* zA=@FXcKEA5v%!qiliKhbL*p0HV|S7IZbr-JE4>!aek1L~XnmmdQj9suz5C^#c;h@a z7mjvBL4JlSgTsP5H~7ksUY{9RQ^B-rgQKhOOV$?ij~i~@>toQ+azM8p2WV{Cq4yKz zk}S-kXS;7Y@#r1f?6pdOlNRC|g`6LM!CQ%L;L%9t+IyO)kb19@krT*RB&84%ikeEV zkFr+@O7gU7asyg-Ht~viM!j0A1W3=H%B>k1Ns&xfF=&kEQV?W;FQhP1G7(y#4(5G7 z()2MRO0M6{BNmX)()`Q{-okNNm<-YWv6HF>uNOXmP)5fLd*uYqR{NKOJGwX*w^?W&DGo z|3I5F7wBYgDroCVCDo_K<7uz`6CD_vnSdlBp=n7RV3z1SZ`!A4yTl=W;^%}&q}`8y zs1FQ-CPLaMy+VNlH}=RXx3qPs=J)ex%!1Gw8V@H>=CFQg4pLtvJgd;P|K^ygP*(TV z#NmHAlCtLsY|FlTHdN*$FUzT!+7nEcQiXfl8Lb!StP>nc7J|4+idmjT3Qw|(e{-^k zVsJzx)|J+G`}%kHg}Tml+uCLF3Em5w_Bw3i$;eUQuDjfOCWZ&kwTj*N@+~4bhZFrF zQYrZ`%fH(9Z`7;GeFv-l{+v+AdM>kjmc~pTH{%)S7^T4SHdlo-uPxtkXOYD3D2Vhp zj=25xXCR{{5zu(>U`*+A(f48wC)hu5r(d4R(y0ry&#PBqoR36*FBcpJji5fE?t-o3 zV~xp=pBhf6rK)$D`sFogF_a*|pfe#0k=z-B~urFM;Gw=?6iN%ce| z0UTy9>dqyn)m4w)5PZSY<9Wvom{}Nh$aB( zRh2{S`a^?P+j=Eq~E_n$eeA}B!U z{%BBei*H=Z)MD1UWb3Uzl|vU-h!^-MB<+M?hj=J5>V4&m1W#bK#cY7~)}IODf29{3 zT+b*gFv*Vu%&}yQ7Jd*t@@{j^0WHrhaV-lua&(pwY%O+v=-+j^s`R0#q;2?hp$wEg zchPEt0w(9}Eh92>adOxDl`Fe8?+-Ym6Sd_>QR3Ux!^zJ^AZ@iL9u-v2iBBb~wSk zB;n|TecH5F$3lr*fW@bzGvnBjl5WjXZ@d6mB_)0K>ECLoTh1{NZ=He>nfykwaq$Gw zgY)2>D$EMPL*wQ4)a4G=eBoUxGx{MLH@$Mr*6yQac@RXh7cKTQF|W7(93`3l>4N57 z^YwCJX1zs+|6wRV#;{BF>M?|6fqbt*<8al9_rfx?NFPD+$3r!p9xx8}?J~4N&Txe+ z6$>|fE6UnHC9ZHVZh(>l!nPmX3_iLwnS8@&Bqm-(i`ne9STU5qV)(n^Z3E*IqW5FG zB=3{a!^)-dEXR1n zY58wG3Hi;b_WsKB4uggcPMoA4_N7U^1)4Fr`I4z-p|9fqWQs_cgUi(oyKhRw|UB`%5CFcelpDk+LA9#`AUB^a2IA35K8UZ?Zkh^rkya zdV9s?i~HV>(+RL0z#|D5KSFDI+s42`pSQoSY>HtPDn9qQJ|O&-JENuj*a7_S<5D)2 z(Mc&-vhlog@;W)5ct(pOxKdJb62s~i?XQ`9xU2>TcX!?{4|Sg#VzhWP!V~TX_1n82 zp$Wi6$+_bis0~(M#8VsPP4gHv0f}8s;UOMezcf2-a)+(7JeT7`kcQ}!-m z{)M3gt|4ysqLAQ=eC46fJlR(i^oH-QC#PAe1u0q!ONr(i6<}Lmv+ke+C_stBV~={# z?wePcboehhbQQb_IZNA%k*;VGWyv%-lEUWMVAHdI+$%AsmUoki&nemw4GB(LXu_?3 zZ^r+sIF1PsFgnMBOd34uk6FJ$e9#0O7P=Dwp~FC^pSH#XhvihvFU%uDp6T@U+%)Ki zL^vV5p!0@yjH%`4>Ig+WV_%r+`vc~7zxqZP6}Y<<9(f|r7vee7ro8b++JnHx+~2ev zkdl`{dr|W;r~ic6W`FcIER>%&pAfyN^BX>IMuUITD=6SnN2L^eFlDQW^elIMVunY? z8+Y1mjT_pJj9=Ni07Oq#6**T;Y2JQeqh?%^4r>2N8(NLa36BWrDE?E;9*5+-!{Bw^ z2CCvy;S2Nv1pw!*o{*pP*W5}w%@MXcOJsjNaXWn#VBkhf+2WaIYl4ijKS7 z%xyvV9lNyW3OC#Fn=Vr{jwV6rZ5TMgNMJbx35tF2NCF1XP?)=_6|`sk!1DsE9>0>E zcdw5ReIYNX=3PsJ%)O@9nfOL`pJ6IJ>tI-u!%mN&U~0K3DRn0P8tZq&4+%uHjCxzP_HG9Xu7o^XS%$l`*vmAh^6;!N?>u25gP~;a@kb z{DykLEE^$TAHy0W9T$A z385t3N`fBzz1Bltq`U%=ehry^0)m){H(I`2Spt(F)#0(5O*h|0Um;*diFppJWWUE z>u_-O&%GvDDs(l1!BsF4gAq@x%dID6c1U=#^q zjzduaVHQUtgPU7r7KSA`A9!Kq=rA=qgCOC+b3NpDP1f_roiuWkCReWfqK{sD?M{tqdQQerYUOxgH`60z z+aS4jPpW+{)=<%89sLyZrmQQ=V-?g%o*6~qro7GDx{Fm!qV*1AMn3+c(*z`xYg;|r z!BP_4EurFN{&wr{X@5l-z2>fwkUI8HDF|XKEE`#BFDtM)$%JUI1aI0H%agYXV$fL6 zjWKA&~qd#p2D zR)6${yZc>^CYi1Dj_l%vb0sei(6Se5PbSS17ISO&aQj>lH>=scan;-UV3%LB_nTCV zvi<1A;WMOE@jnS!)7M8Y(K&GP!N3`f^Ih(STYi~!vh2;T7$zQUejowJLWg;y&p zgC`993kb>&YKREH$1~w}WEFgh;(KdyA!3DquLU64n8PMN6whd+x7U$_1~6}il4y4{ z?gTg?2*Pr)ha2XnAFr40*$_#5Gn@E{UZ8;K@AWR_>Jz-Prt@J2<67{tSQ0HtMuA;N zO;mtY#aVW-ZOnbful7UXkeaTPUn^#4kPd=kdrR$y^0-rR&x{^Bc43cPX!I$prWYsx z{bm2rqOcu*9`IJ49Qhnl@O$x&HBBtv9mpKsH@Kbi^7calT<;3q%1w$eOq%op1z42` zXyfKBd7RQ~q@GS8g{P0x zW95T$U)b14e>}a^RQ){H5lt^pfU)WIm(z8_-rPU>BEpols^3(V(}G>J>vF($eZ0o; z)>Mts(V2$}njxZ76^CPKxh%9798a05AX7iOAh2V4cv7^JBi>s5-0hAB*J;0w67|#4 zD=GkK-v3YxS>?H4`1#wc+os!vRj3Yw^g&{YC=wD4otEZz3nVP;`H&-iedN*k=)19& zUqa{CfGB{8c>nv9d~{_jyS8{nXQNuC!^S&t4**azOGX-VY|jD z4v$((qJUP1GL`}B**7v7=iz!|8MA8PCA9A!FqoCA0+^>4ChvU-OxftC&0@ZuXd{~f5;va8*k^z`5E;*WMjLv z`-*a>L*TMRpajEtYud;CQWw!a8kN2-RXP2RhI#w03-R#+uaHB+!Oryky51=vL(+SzFR?1 z;0!$xYTC`MJ;B&^jghevo|Q_9^1Ci(KhL2-`r95wvAa%w*qoL(4uWSMR9N$0F};Rd z+SWfsg3jia-Z6?7FuvmR&@Ca=!kcz7^735H z{bd$Na|(_4&8p2vYWCmlbrf^!rCr}r?sKT?5TMu=6P1_=gEw%M|ItWO?lWz_LS-wf z*<#GS@tfWS(MfOasK#{&%^Q%Xs#Y5m5YfeXG`j}ZoUM$nF|{`JPu%;tnQACGTuLeo ztV6gUCFKTda0@}gt~_X znAMhYWpw87?+40>bFfkktqo_lMm)OH(%1D5LjlJl8}sowgi;}mr2W-Hbz3enpRHZZ zd}tl)n|8!*VET3<|Gk62HD01SK3_L|O^O1d&82g$5+tW12E&Ha3WhVo`%@O_DJ8!O zR!I_>?>O*2qk^@pug$x8xAj}uAwfC^)^sOS*=W}dpZ0FP_?(GnzuvVCS(&W}cgnIl zDcBtO(pk4{%&RPXLU>9!)nJ>#3V~uj93J|z4{DTbLlbPDd!sB`^d!#hvX<)+LG7qD zDGHc~&uF}G*`&0@W!NhGg5FZ+nNph-3QZ(h`k_~W?}h2(ySsM1Z2j|Q!u5Wh!*KN~ zh=x{MuE{qWJTtsRO{|`&1b75C?o3pgVo=ZvT5plUAAj&;Tqf(CZxB0Ai2x?|;A|XP}$fwX@tEty~sQg(?_*}0&SDwheJVu2A;3CE*y6sJZhFu%~?8>o|@4eXR zNLAkC<%`4f(OZnGJ-zY>_bVEuBJ3we9G2&^E@yl{34151^ zh~4o5-T=cjAPOMWZghPS|EVLuwqfXC=;L2cwoB^#g;1+XhS^VL*X|27l1;2{Z)Zo! z_Ma4?7bw6t&@L#uJZ4%;?|vDx+Pn~F(W|Bv0Uf|#;HnONMw8N znJ@3vhBK=mbYa@8C3XH0e1lV6T71;5yvi20Q|ne66yWeocSCU6+55#En9HeSHZuHd z{O?x`3QvpjMu@f~MAKo-0r-H-02+%cnV^F^5^VnT9EnowiX3kI{0(<*!GjZZ}H`DG<-TA z&~j#gd4)XH%X>UANkSpKmXq~u z#C(lsh239}P;=&EJ8CB+< zIG3635MW~s&4Ho3%W}U*B#ZuX+xm_z_-)5W-zoL`i5ecEj0y#`!SWqMxGFZz*vUw{ zFdgK1=DO?NigE`R7fOK6BUF@X-Msg*@%zVfgXZE26o z7ZA%1lq#l01Vj?Sg$yP8vjsHkq$5k?2`6VaihgPz!XH}3)!OPnb324ys{?8|UdS8FA zp=eb8*z}N7}9rq3s8x20w0Td9x`G}3{NO9D$Hg1_u zZ0m$BvW~c{04P1L;*NJr-o&3Kv5qL@>_4>MM&YnN_3=nAa7BW?)fd_&^TM<+?Eo_H zh*jXlufB~Hf*Q0+pqvW= z?^MUs@;o=m%_s$9KG{gnJL+}S-9grwB>wO#liilDuSVCtLaWP~tzy)e=Am|GRcddK z6X(6r+?eE7!%LkT~@2I!Rx`9f>7?V-*K| zSQ7c(=bcMy5f&EGJYQhC&L#Yp(G0y^eNf4)LWTQvbndP>c=cFqYhqJYgD)$jJ0ylD;c(mr9{e{WP{(r9i~ zb$=(LZl{E4_|^@_4ft!HeKM~{c07u0pivZMtEauA64=C@gs**XBc(reXl?KKhoJyn9`++nNl{muN?bK(2k#bDD(u@z$H0ng zi`XZuvR&11TK^TePwi4E@6qz77bsxQ&uwO*srBOP=PN3-zNa{d$??z<0%R;2oYhF^ zt_(@Zy<1px8%c@b5kbO6IWvi={-Ot>$Sam|_@QoI4{nqNwU6<+3+(6}Dsn?jCO#cc zUkzz9fUr40Q1Cb}ittp`n^?hxZtC<=TT7yVrj$uDZG4^ZM;UbmtGTxy_CM&LrJ|_c z*-(=@>Rk^M!q_J3JwilAj}?<0+Ea&U9|@3&@a4}cbF(Z$%4D<3J(%|X;mzjr6*L!I zAxHwXv#c$W1~iYILt8vfbk1+#_H>rw3}jR&pi)2}ZmQf>YoyKkLA`VtcWk2}il#_z z>R>&cc#oX z65_Sz67;#B>(!U?SsUJzVb-qe4WojU6dX?nt3ta+UZBb}p?c^QA`*-0CENE#;UjiJzJTq!b`GIoQK09Fl+j`DN`?~h9_2X6$c(`@K z5u9AnP&>BRop9dy+1}RO%pYBTI=*M-MgnIYXyNd@!_a7O1^Z2XX7Wvc%2T~kW zY5z{iev`-$Qy-N);i>YKeX{CM9ztY;i)ln`>pu(yqzxYTEv=Orw$rsfyDgWad-$LZ z?ULP(`VBq_NpM?U44B5v3O5IhtNSIU)pY9paeK@5RP6s?6kusBuaVC6T^V8dTQFZT z4p$!DI=H;CQ7pj2Y5n|%wcM$(O?TIE6lHxF5*tGQ56w1ALgxg^S9qKOM*z70_;2)Y zu+k>8_9l`E6ik<(SL6{{`F4Exa8A{(7#iPAjkBCElmZuNa{%U@;+wM3)Wruj-I z%#bO0d@gU=e>_d|8XiZ4Bg@_imlU99fqT-ibdXUFD4?TL$ULFaWU_-hAbhyFftL?S zQId<7#0CIGSYCl(o;ew+GJN~;>9YlX7tET060AJ$?NfzA@fg5a!eB+2I4$i=%Da+n zNwDov{YYtmS@;)vQ{jp=s?N1R#-Vp5YSs2A>gwC(lpz!}AxkNV5-GVze@t>_s^HLJ z73~Ev?CKyk11ZcU#tj&TOIFsgB%v79dL3fzXM(mCmKddg zCkT|ili<3D*G_@@9@_5BabuqW66>M>M~o3TUCKRAb3{dmluTBaFq%5|wU_<$6meVk zTH%bk)`-}gV03xm<9?~mPoR;HZNvjcI8_MKv7^_Tai#Mk`n^Tlbq8%b3-)xndr^&W z98qxdJdqW9#01w>plrGh@_FhwlCK2&h0u0e=~&KyF|XNM=!!mB^_@+e_okp%oRVhe z_bqp;0opdJkk`%D_H?l1u=Y1$qyz5yZ18JtwGyYEh9XX@zsPO|>eDRNiaI61nKLb1 zHn58s@@OqC$74$7z!@c;*YPX@mo{54&!abw*=mjF(oJUXxSZVH)=lq%SMB!0as(zy zcHIK|GGWq{ugt;O1qUa~`NtA!!C%vLUjC)^3V582oC`QqUHktjL{U1SNK&FQm>DyO za;W4Gi4sYSnZYn-GzW*Iq=*tJ=Y$A_L}$^7l!{Uk)ss?0PB|Bn|K58JdwHJscU}MI zect!2>-v7~wf9>0y4St#wf10jbcnjTDq3DN4vE8I(LA^uDqBP`x!T+%G-I<^tDN_Z zZF}tw=wPg{sFLQzV6vzpp)aoC>WIx6l?3BdhOxjr|tW%{E&^nn$0u_^E$AfKSm1k2WkYygAi02`Tgl1Qn8( z7mMme;!rUxDw|8^u(jCUB8_)%yM|uOD0leg{qC@DFZqEvLwC? zdfh;(hd_AD-a*}()p~*zaYQUPBS-Um)+G^s0Z6OXhQWF|?zS#FjO^rxGPk&2 zm&EG>D##>;9l(H)jA9G~A0i`#G<4=KgUeuB>YG>m4msQaw z3Q-DW_HX{w>*_%NpfZtw_-}o3-z_$(Es&bGl9Rk}m9^(zen-M0e-el6jqzhrsIWn9 zPM^jG%i&K|cf}{lQqOm|uI2Uv4YQ~u3Wmd?QZarQd>|8)doZfE-;o6tw6SbQPV!-q=dXpu$!#uf}@?!R#= z?ZqWR!h3~37Tl4?{D4BJ-&VauFpeFvw#K7 zfEJHICj%^33S6RfU7>!K_vz@aVeRc_)3r0&0J$!P%LrsaGsuHyeWh8^?x@SjzRpUw zBhIR4N@12&jskK5h7HXi5*;xu^?TRVjp;4#XVz#8jhSavnr!&zGoU46D3C+L!i&x0 zaTjW?Kl178wagi=UbBBpO#u~RbucUvIT(INSlqTJ!=w#ozj@~mE3w6?*F>|<98W{V zI2{a`MfZe-wyc|2lY4}Z$Mb5~y_Y-oMKhE>0YWT>%wz;mS!@~;@z=Pj^MNm)$ek)a zJE!kggmY-oSF0RAjzjE1uD~GqArC#!`ZkJ|z1OF-{-%HTwz(QttH;FwJ)Wm0`N7(P zUe?YThLJhamgjFjE}=cEKPejlXm$Bo7TKE?fEa&TkK4rg@p%U}_e9#4=bsO?{WG%z z1@Jl;HZ+F*Y=06Nk=K^wydd2-ycs`}K)JSp<+H6J=@cs0)q!RRi%rEa{W;LfB4blt zv*KrROTbSDtw*n$#6B$UOeTn+1VIM_yAwN@&7mSM5Urv4>cCR=+narod(DWYDmK+K zY5+L_<3SB(GAI~!upgbq@I{)CW{!AWczf9164QLHKINKAF+1c6$`Ij);&@}Y92(DC z99}Iy^|`c`pDT{upEr3fz_vI75Ox}b<3s9-k6RB*FR074o4Po%eQDD!7sFlv!cBv?LTsPO z%W9p^vo76!7}cPtQG_ul{6`VM@BjwAmnSUc&pr)6 zy8mh`5SbPO=d9LRjYm%|Fne88TlypQ`=uB^03rYov@}tXId?2|zW4gz)FZi6Q=h26 zr_Wqx1vvl=5kn?%FoFQoe0y-J;-Oi{(jRZ*x;Gsk3F}-&0>pY4ZxY)Z!w;A6LXpz8 z<$)GQW2@BXop0(e#v&4>>407zcpxS;U+KIK8~ZXN;-jaU*fo&}wfScemt(fb)ooRHmR#Dn){XE*ru&LehzCr+YNix&V z4_YC4?}ARo$EMpy0{+arzH*zxy2-kEnz7H*rGy*~;y7HW2^!Duy}s++J|cR$_4aE_ zp?{re^M;Z(GocudxUxSJj%}VaFIa!{_8Jq}I_JhsC*4&dV`%B3vbz!iJr=|ECb6g# z1cUW0W^a_vsBAIabN{o^5jkRbJ=gXopvUp-1A+6Dj>{&NTg>R(L!C1|FIoAwEVip1 z0PvVV64^Hxcp^zr`6m_Sq*)OLL+P({U(f@SZL|TaQ!K~%m{JVop(2kxq0?H zbiY#kaZ!tq zx51)%$R+4ZHZ&8MP^g0C|74hG?%HjWqAa(}R=>ks{_*@_bgt>bQI<{ig@xIxal=Qy zN<^9!Naa>&OnzCWnouSb5|BX+;01A5ZP7O?@H}PF!Hsutdkl>egMas!HUnxbCXmSz zdhux9Z-Wb`+NuKgU8W9D>fg*;cmEt9#-XNp{|osgjJ1o2hj}n zk(ZsHmEdaJ27db+FE8~_MFD|8YYd9Ws_p5LDrQVsIWe=4xU6^p-*-_Q^sI5?D( zy{TkhCYJ*RCFqYBG6Z@fj=k}t>u%*sIP%ECja+%L!a2ENH2{eHKf#m&cIV2oySlSd z)HZ&8?fAA!&H5_<;{FYCVaT7T6vj}zs<`@y)KR@{4&9br+86-B{~Pl2h57^rXI%W_ zO3XSXYhI?mEjW`FVDdrQ765hs4MHolKg7-+6cEwa-HkkKhO06 zNFRWZI0-WwrA}N|Ub4Bcv;D~4c`H)4dd%@u02l*+fp$AIh9?q|hAJ1U96zMKSmOS| zo6qkycAREP$D-@F@BrxIvuRMNz_UahVW5-wD?Qsm~Fg$fD zGRzMk@Y6a2+Np4PYvp84|J6e!f3Akgkfvm_wJ^N}roI5Ci(v=SpuG(;w)4yZ#Qp5i(*aJkALAFpZ zVxy#$A(g6dbIMjL!Kq`$PUf~GaTI|<7QDrSMiZKubZCTl>FVu;%VH!y49>%yY8rca zV7@}W?hf=duxLc*d8`MC0;hg3h#8t}IXOzrzKQ)r4Drv;_v+hF0f1Nm$a7g1mG@Ve zq#QXmnL=1_fA|>XnPWW@_NtdFz71;+793Fr+r&>N zc?AKCAC1k10vF8VS0{8dgdH7!eR88iTg#w#qiw=UArG(CBnkzCK4`oOSHdvWyx~OV zuhlDFgjEbabVvo{SU`^91p8CT-Xw+>LTbhNud)}rXY(}HYf!z@TfepLiVHyE5EA+? zi|WVZjf2mT9TMBZ9q-M&Eau*=_%2yYW3V3}@CZR;aG)d_mOJIFNmAasIzw^hE~k&t z(iu8>JGKhta4JEDGAJhWuRom{Kt-|-L(wZ*Hm&w=4DW|~HI(K)pCs*z5+Hau`qJ6F ztpkS1@uu>EeB>JCqp4=!kF6Vcb7LOn@WExLtTF)x`)>wA4ILj+~9v zKC|tw$F-`N9~(>S02YU`6ewB?-0HYkiLIik9pS(M(PvAm&y@~pmdBzL7I;cu8l4U$ zKhTV%dD5uJ{4WaQ0GP^RF`B7P3aAHH|Lg^E4%+48S->W; z-!r0b-l?Gk7TzzDec^-@v2Uugr0D9iNb@}9z$9_?ezQwsK2L;dcVS=t|zezqEr zV);@yFXQkxCB=E(&d197TX78*W>rP0OQS1fZmq&NZq@B)DCOGC07%4Wuf>sNcdfB3gBs~!f)T-zJ*8qn(s^l)%vk;sCD zX2+Z zyZze%h={%nn@!@qPQ;&sR=;1~yE*%CcEUAIxmIvgY6yVn2@n#rG9!b%Cbc74Z)oY3 zA78u9xY3G426o#S1AxA;4KAHS!$4~n(oCynV^F8(`sV!ydRnk^N^a$fRAvK|fpBEt zojWUl-`~jiuL+gg8Bn%Sq4ebNx|-!%Ow*m@ARB!;FoHirUQDRmv}RoNJ=Vol4}<-_>(3vG&~9IrEgU4?YIZ% zb^r5(&_qWv%)Z(5+R00}C#v`cr{S|3(;Eskg#|P`fw!q(@izG%K31&FJm-J$oAI7Y zYInx|F6w)<3XsCR0Bwjkd;;T7U)Oh>zy++&nz*ubmFtzb*{+eK)c{7AhC!PS4$>X%%65A)%VzY1ZtES#iEGcy zvL$H=DIy$KkOE>rY5A5dJ_?r}RpA_I4oS7^s5zH~GCf|>giNIFc@L8E_Plp8!(?e* zOm_Qz{6iw3!wNGPV3K=GQd>`L+B#p}@1v6HPw|lB8L4XlhLAP4pRRiA$cCNME&mWzUvxA7Ck{zZr*P1t(ewAr2(X*Vf7jO zk`Am_qi;(V-BOV0+#cIn@?o3Dkz4QEc?TJYTx3*k=FY4)ubY$1<&OBj{6AM z!(#OH2?iLf7AJ@!qF=ex=kVpcO*7|tPx`%HeJ-6MAvsNpv@J~9vAKbBbj|4*jcYSS z?pB8EpJd2%Pm^;wp8AMkH+%8A8S8Q`Gw!r^v)8Pve|xF$IH1H~_H&sWUKn)p7GI?n zrq?Yk|Lg0S^G>}=ayL1E6c2CFcpKzhi+)+_6mb5o8(j2bURv|;2aSL>K&i`9qK`Xt z$nZ&z-S@DodTW(rsiy0nOBmJ)^l*=1kOM3@Ad`@u_4{NV@2bALe2Iee#kc-EKz*4$ggWQ?KFlq`|`5HRq|rfLu>7Qk(!@66gL&Wl>|X>%I)RlPBHP zPo4gGwifVCEZ-rZypk-KXfCop0&*s$cWoaWUnF8n(_NqOL>D0Vz7WVpHcH4r!iB%s z#Y$_3B6fB?kDK}R(btY+aVUV}y8<-`ith~GL*=_`j;(yZX9q>k;pOIW>MIkUyXnG? z$LWZqu0O7L_$ai6dZVFy&};>5yGm>aU=cg5<1q}bp9i$uA@R-s91i>Z)cHItE5z!` zgtocZdA%P1i4$~>(DTK7cSYpqwVm`4kj1avk2N2k4BMU@@O9@XTRQFdKdPnKxtdvm- zns8j^4{&^sK(Y^Bzr!0vs9l(C`}Q^NY1E|LQ(Q_`Rm8yc)Pw|OfW-@g9F5B0y_a&S zW%TDcQyp*T&vy^?`rvc=(ZmQq@O>hX$)X_Lw?*>jpMts#6>(QejY7V>!wx?)9{}V8 z!Q`j$j_f+NcABYUM$WoL8VA+>F~lyXsGbDm#OY%NpV~t&1rHev-`zA2VQH>YJ+JbD z_vce}TOM9BQv_f^lUw*C$R@Dgpr*&yO9DUqijkn|_+V zzAUYLtSa}%^&78^#7)Uq zS>I?5H}L)XYpabbU=GVS$6zv$fb?zi1`(SX^vKB|qdVWf)f=mNc?F^7up%z)d5Hm6 zv4Igw)^uM=KmVqIzHT{S4#zi#W-E)6lk82a9DQ;w*rqTW^K>S3j|Bq&o)4gIdS%Uh}PbNjx1i+X?K{XujL3`D{$@kLJ;yiYnwR!}?sJHp50@Xw{FO{~yE$RyHTerx*1;P3G9KBA7XV+NFD~vO&?NZI*eUW z&~vJc8?}g*QG8q8nCht6ln4-lF{Lsn{Gzhw_zq{~+kJ7e+RY4|cB{b;8*^jz>td}17f}Ew7-WC`f#%V-cfC^VDvQ-DD*n7QR#iP**ZCBX3#K_7b&w&vIp>eflcEj;W9?cTpJ$_xurk0E!_Sh_pTQ4QxgCR z#-8oX;txuS>F$2ROoz89qM~KmX$k0D`d(WMf$Tg?PMDYQtfb#u>xQqg^%Y z{?zRwXRHGV0!j#GwBc4oieh$yy3Omt=!9btQ!#CyKLP|192Hpq@>YK4mzw7;t1q`& ztk~r^SfFcL2IvLD&W2MRPh=9U51-L>c7EwIQKu_1^Ibor*tt)31B77Q1x1QDpDv4L z^A*L0Y%wy${^_cVS-HOEsn_f)@-pTq$Q` zkXktqWe<*HJdRHtq|bXgQQ66Sr&|6@^d$fbhToIqOU00&6`IZArAY~TMwhcHXK$un z3X1)?p~)Fq1uy`I&V4^7KaDP5{@0NH*K)qzb<4AVwG4YxPoJ7X34#v)PFQ%7yS?tG zn$^j<%f4it^CuE+JR1gN&;$9Q<-&FGm_QBVd&UpfhMxC*?m5@=?42s17>87E1!bJP ziWw(%&)>bKA=X+o=8bcd%$(8RfE2#(0U71re=u=Lo*&P>@Nk@a`J>L0(!yEv$F_hN zhvDm#6E$;PmJgVVh&nW>3@lSToUJ|!(BUzDP}b&0LQ1u1p^BS5b@egv8a{Lh&1V%K z<}?@sT3vq1K||JZv0ryP?TiD&2TY82)jzOFe{CMW0TmN)a4!HiWkzQ{D#g>SM55vY z_59B5m~}9{vPclAA+<>F$dmTvBVW|N)=Rg>So?^&&*~Od3J7pzfkR>mk0{)ZmB!`v z52T)UdHP4pk#$>ykS_^{v4RVE$c;(lD8v_guEAqBe)+7%;@0fBP3? z^~Ha_=GJx^KYIqtVs<5f;CM|4uSRWh?+C?)OkNE!33hV6k+m}SPWnPXj_1kAOcsla zj0k*m1HG>xpWT6byHLuTrdYao2-)R62jXR6WD`OjZ9xv<}y=`HD-MeqY z!%X7*op~{Uk^nstasY_oSK>-Ykyghyk$pP2nZun^s}oEmpf4*2pA@3>D;(Pmtmde>7}w!l@Q!$-b8eA%JXp_2fE=P|-#d;#X5?#cR-Hi^#RwP&Lp9u z)0}xz@d~l%{nbB_YwaSvtgbk1eGfozp_4}T6`pd+>gBKe%i%ItEi}&x93JwYzunuZ_YyS5?$p z6uY1NZm~#b>s!(`UqFf6UK5nmDx+~Le3#Wn8&*|iUm?Ax{Tn?J|_YCM005HEuD8JtLUYh#h%7pd!C)Ned}wVE8O59kSeJFW2aI!8ySX1D$kN#~&2crf_U_6XJOcEZBLVk+k_b`R-nZWEi(YX@c>{45Lw<-ziP@?xr5uoKDbL}=a}Bo zCKSNyfb$3=tlQ-!-3eAaSg^Tx-D^LZe}4Q>_9=LohQE7&DJ+IUrTU}ABH@rwo8NLO zo)ql4ap8&cSB)YqQj@Y6+M2Eo@_mO;J)HfDF29uQ7`B~M#N0jWlM{DwJE|t|)k625 zOT(Gl&+u*O#-5tl`m1^DQROlLBn^b}f@p+8bd3@(zN^hvylw6tv5Um{|n z{b5tkvl~bMvH%Rr!(io#t&VKPt4lI`?8XXLhF!Yx;NgSe7CrD(}3c4G5MwFkmv~?3!0ly#>QVem1N`HX)@}%JAP$F-#2S>Bj(&J2N?b{ zK{JQU%Q2VSeAs9|e$t$J^IOtU9ObJM(M1E$3!V&c6!ma7H|+D3vR~rf?AjGwaQwK& zwKD*ri}BzxD7@XprD4+ylJurEFaPZFJeI#gdy#mn7$7C^jpMwK?}jq%7jG+4tZCSz z9v>EPpe&__w89@y6ZvZN7RJz6zPsZaxBSMoaKmNd-H&>w%xh5qif?cRg-#$C`MF>H z(e;G&mgZ-Q!qj_ka*{cU!Yks?K|?HVl-ef# z+|E(QGi^t!D8S%RhQst{Vwfy`ZB~;#_#tHWa>qH7rh2|kp-SrkuSYcud-nd~$fTT_l#*UV)=%FLFnT~Uz(KI!*1vV+{Ff?c#w6Bub$h&A za(Tfrmv%jX)<@}SCFRFg)3fq>M_0-`c^>;n(qgCc{y!Z6W&rjHi+^pc=CV&(s`Zr? zcg>T(@>}Y%EV9JaPzEcwWhKZ4MK{0ibb8bLKwbr}WIqs=(ZD!T0>}ZcKrS$OK*0}9 z3EP!#EVoYNzPY*Vl#QC(NZ|p==qmsW_y?Qj=TGO~gWfTx%qw%vBZ)1~W<2kjA+aMr zxj-AB01x3;TKI3r%Z~nNOMJN*y=Qa|k#x`9dtvP^x;4m{u}zrV9BC@LDnYaC9QMI*PQpsbtwhpy6B}n^mfMeiBac= zGmGvwMeK@Mktkw*=lRQKfFbak056|zGP3y+CtdSKrIC2zTA|ETi{7em03q@b!H!cg zE8BkMY>oB4*UUGhI2&&i;_>$Kza6z+~UE@-t0HDtY zxc>0PPQew`-$UP~G(|p}$L*eZ@kh7XdS@PSnCDbmY5F!_|7adp)V0fXA z@jeygzIVs8y+*@9e#2> z+vQf%-;#U)6nFr9UmEIfAPsX#nwX<^$AE1X%xoz8`)pghgyBU1Bm(XRZMfmH88Q^k z5Er_`StREao4Fyh<6#ydZBA_dtFSF73guM7y`7*GA=28Cq~7^;o{h%01!^-cCcIHd zZAS?L^2rdLUyeU{vRO$gmGU~OX0f){Y?m#a?9`n?EtbEA3O^SS1d##p@v>Nq<@y`Z z4O_fOqJI`j(N$3jKG&tPp$YB5yQOeQQpusOYMK5qo7jnS2^wc*LxhK+M7Wsc=_l9* z`>~M%fZ6EO5c_WOtQ8)?qKfl1>CRV=hY1PfvLlm)p^~6vo&O0?n0<{;!w#x=z@BT1 zhQpSa)YQry699T}Cg6u06w2QD{n+~FN86G)2F|~)rQa_w_IIz@)(PX+J|4|j89G^i?LAvgSDfYWHKGpnjyL|5`q4{-xzAUOF>Rbbp7%(ZsD1(Y~c+3D7=xsgoux+EU?L;K@} zYJKfziGUK%Q$mY2ulQ;9U`LGi_n)(B?l1kdh-IjKdiHK3K(31#9w7FG+|-fsd|37D zjhal2Jo#g0H=rc&4GZsUJR1IEL}lOcv}D^CWi*Vn%21_*`LiBeRwW}-t$R}UGf^Tg zO47t9rC~pN`5n6j!jHf8kSq1@Mg&&Qk71N6tlRt6f3Cv@pF~BheZ1vWp%N)J33pS% zn@aOEYw7(940gP~R~{vWjf`8<;$jOhSb$-3{rvcA0{s!);4ioDd|`rib+Mkedc*Cm z%>ac1DD)EW*piwCn>IzQs=tzz@}D$40$v?s0T3Q+Dwu&h9ao=AIsU%P0=wzDzU;Pl zR~H%_hzAe?fC!h+v=zUOHst>Okn1-{TT&P+=6GD|G{6u62K}T0gMa_C$5`8ap?C-J z+d^-TrFG)R07wsjpuhmf1*EQGA87Z|>PUs$$rBCDCGni$@loq4plI6OL}1mRbmur#)3-sK8bP>N!j+-@bO*p}G|4W@$03@JtAjJvbX z{*4}uKUv;$e&vqf$4^h5159B7Q(u2NsK(ZkJg)xpGD!Bk8Dp{a!2&>v1Eh|1k?u?W zsoCG&l6P#NIrz`uqdUg{DIUJ=D){tNP#}!Z7~OViQT(KldgjbI%oBYoc6Gl1MpvMR z0ny@#Gu}xGp((mH>tq<{}g>nRmYid7DG>-Ig-e%dIN8`7;R~ z1$i{=kwO3yq%dSIl(hSSSJzvT5AM^Rqb|L>yz6|y@2^@vKS#I&AVJt3n@Q*8WYT4? zm#^60vgDe(fvh}s-rSBezs~^#aavO_V1IGPr`_R&PS!cL?84$9%s#xD9smfkSorI1 z;bkk!;a2Cgy70_TXQh{)bE%sBbcy%|fDt6J{1-vGK%>zsxir*e(du_ks~CD^j_l>j zCQ(GNFXS+p$gZiwyQ&*warvtDGxarjm7=Rh57l@I)q1?|sQJ~+;R5lkc1xH1-91yR zQ^#)m*s+nzmjNl3FU4>f{QpB@j^HOVTnaz0{;qZw=Uq0xmT_eCqQ$RY!n@^q`uy9j zf~V^K=T(|=*;u)y!k;}Yo%ilcdao9qN$DeJQv4r+fyG;1xOZ901q75FXsy}$(ERY) zeH&Dto^YdZngZ&oF`?9*uHXD1wm)i%ur<~dZDDkLrVWrzL`jw9ylK5Ht*a)jdSe~~YZC;6gv~J9aIHa;R-d%ZISfDn5-yAu7pu$INu(=6$pG@(4 zG)i^Uz}<|UsvWyuuM}QnHh}&h_$eB6@!UOkrgIxRL;j=!G3ubhr>0n=*oOdt6%zg| zCa)5ICL#4lQU(4!<8Dcbaz#>ozCK--B7b`{d@CLG)&_VzA@4`?V)0%0v|_B(#1l_3 zN34?AjXN0Y+cQctP#JKohQE;f&ND6hiW`gv@6_;@MHL1^{4yHiCRPBfzQ@ z#>UKZ++-G(&GY&sl6NFD@O?CjVNrMC!XF(4_@Pda#n|Q&Yn@`()dh1K^JBWjIPrl+ zfFAWq_{Wye+W+mJ8EVnH>^eEA_FsEK4!2g<0_!dR>5Bh)Zob%G6 zt#3x|`j7XkJl6njiZuXQYJT86@$;-tlT|m!)I15d_DVItI_ixiF2|d8-HPQb{2{Sj)PiB7V2B>|(iTNcHf+8J{o`OtQ$8~0(bsYy2ZKOxOVYFcfbbm-{F`-rBgY`&#I6p6fnla#(H}wbCbhpof_L09DXk=7JP5^ zNG;mWM3HI@iDhrvGuRz7LqFxOTl}nkr^|hyG1O=IpI{0)54LSp7+bptle}c|XbEMr zTXu4-Y_1BxqK-@UqX6}autSzs>xa9elIOo@DnF(2$bA=E6uyIiq$;0y&i+S75-)t81JKGlJ@jCx<4Kz@V6n->1{Kf=7;Be}uQX+dxVdOVc07N~O zcViH!p{1?LbD_*v++y-k1MAlN_Gd$BS_eR)-sr(%`cfGnO5b+6gqZNA`$uGS2xs%= ziHCFc1CESGy^=v?!=niQRqz8jLsfIhw?~fEeT)w_6!q^HYxZ=I0a(--`PW^M4_jbR zYD%{5oJ5rU>iy53^=U>$UUCZF3V?tc!a0Dz19`74=q>xO_JKC>!OKLI8%a-WBRa50 z2G$Eb2Tu^W_+53vkhJ=hhKcW0!or{2qOd6`pba80-N=w@a{jM&*c;@&T-!IlXrPoJ_X0O)%&s|JAroe{xc)_c>?`7&%9qWs&#wzI_F4Fy~Oxz6=T(wtnfYI0EoIO zgULX?eET2nI;5AFUi&o0M9TJEv*IT^4fpmh>rflIIzT`13r4dD_I}nE2XCGEB2w{o zzVDA^)S0^g0y{m3_|vJ=uGSdUdnt79Qcw6|^;gop{*SW4p2PogaO20{<$GoOBkA; zKmQJ7xMIi4Q)hm6y{;%o@Nn;JS{!7)0MKK3dR{BAoiR4$ z2+x@ebkXNRz63@bGk}-JQpUD6xjgir_jF&(!Q<}_Js$e5wHW}h{{jVv2t^mA6_@vq z?WSsT^@(c4(1=xZew`UHvtoZiUK`js6KG@=to2P$xq!Lmy7K1IckQ%+_- z*k!Ho{KvA61*l!T$l%Z9V+-DWr!}?L=SO6o`x*E_OZZy>25>8P^mVdM0 zG}2xcK9E25{{S1u3cu#;0f4+uc${sPc_7r=`^RU9Y*{~L8Oy||>`RvHYsyZWrNkJ+ zOlHQIg=~!_x+Pgo`cjrGNknN#_N`Kg(PC+_i$u2&Nv_}cUUl!U-*5hyf6nW?pXWTE z=e*DJE$BKa#dx1Z^F6+hZlRZvk#`=6hwdeO$Qr$?rKzc>qk~i@qVWVGnhev|32A6H z3r=)^ACMW$Bu&^dtcp`<`cV8q1&1FI&H|U|&mA511;iaTjMhnQ5h0pDrmfTABNh%0uyO{S=i7A8Up1OfkR!NS^6RGMhE2o~9f3F;19y6l|wBXtXYGsMLqnNimBpXsJ_V!P8nQ9x?Y;H#TQpso< zP8|cY|HPj+8`NVvop{cf&%)ahDH6#CCg8Uq{*J6d&I{d{ZR+dlTjW#j8Y!TvK^}*q ztd1u8heT6|uqq~}@VSCM)#lUkOu}B{h~iN>y8C_dM+*XfKX#rsYe=6KHsn8k&N^;X|4F-K%?9JB*H)KEFQ?Q>W5)qn{zd&0~i}+0iL%^a* zY!R+@Oy?~6nMK2RC8|{s({fbUn+3|b#i2>b(|K;abNi_4TW<6Fyii%Ky0+wy`-HzV~sSB!@C1wzDUjwWwJQz zvuuh~zZ`G1zSuGap75GNYy0_INXLqFDjdG=T|VQQX0gj^07&r}IN@kCvYJ1Z_TO4A zyf>k6`a5<{}RxvyYPgC>rWy5yIU>Iq4SpUj^wMeplbAZbEsu&Q= zw;f6XE%}8ZkItv|hdpz}wV}_iFZ+Y5e1%qNnDFM>X*K>o#saD+H>BuFS+7t0Cckmr zsAK}3#tu;C9q3`Q=ndBsGAT*aXQ4iYndX2Perc#^RwKNccrSeQzTxeKx*~ojDsu@` z@ZW|Ohp6APlN<_Ph;%0OrHI*d-|fu>!2;$`Ra#rXy)O7Emp4S)%a?Lin*@_Jz#{<% zDCnjv^@_?{0mQDp1_!Xa@7(mo`@IejQFf9GMV(i}3u?tAGJsD|79zEe=l9{X4)|P{KT2 z@}k#-Fs121+46k$wL0rhkFmFZONW9b2GKV1fn8`CNz)+i@GY-jT{6107&TFj1ucSB zkayncYyp*&KXax{co6|g8XBM9>;e)(U5o>(xe+B-2b;;GMT4#THTm<+RVRN_r{aPr zG(u1STmRHP(N7Zj9f(%fC3@_~^6>z#t6%>w9s17|g*O&d#Z{TvYUY)S_t_(Yo!xn} z{=ETUgfpQVQbgH&Nc4-V!li>5_AOe%Kk5lUSi}i3!SYF46duaJ#3c8L)trp;Pi$%d z6cH+r5jBVMl0G}EIC%aC`K_x>CrIYe)-(6hK(MGZ#9+L(T&XVItufF1Zr7V(7xqPZ z1muZIdl{m-W84}eEqMpDI6?r{C<)sk_P?`N$H9=6QA(KaiY?+1EYF&6jeAmRmTP}O zBtODNnJ4a*WZ5Y*s&?*-jk7A*U3#FG5(cEi;viKGuC|d24^I?)-r+v3_n`!AWZMuYNXh4D*-dI>$(7Z=IaTiO}*o&h1ZL{UZ~CrEJJZ?D!B`U7;Tf zGi$$){h>1Q#Zfy2?Ozl*jEzh>ZofT8Pw%RL6z&d*%G7X4^)3Lt9R<*55xlBKF87!& zLH^i!GQ#$h)ch+jwx_s$_eJ%E0Ga#DMt$7H(0crU+6`)-<43enF+w|N>Ymr{Iy(TTUQq-xRxCfdY#Nu7+P5MZc=rsFXt?+6Uk&hs^O@~czB>mr~G zDaCL;LEH8fD{**|UW?O1AIek_+G4W#(E|Q!_R&%2JVyvqbxVi`HjgShJze8Z*8U5i zlCcM<&&Y`i+V|%_V^PrL4a=JF59Q$LH@PHtynU%+ z_n!On)sO4Ig`J0?qcM8Mj&Cyl&|G<19dg$#R%GjnGWfEy0rbfqg0`m-lZfKIox=>y zhi=pB8QSCp4AWS_0S<8mA9EaL>Y*ZIajQem5@-Xp&rF$FC^{Sx?oT!1PC=#56pt7v zmIW(j*2i)H#a(_zMZDu5d%5Q_yz{xe9^S!XCkw)g{yqjVC>&29pxG{q)`E>rDyRp& zi%GR}WDcG^HL5CDMZ5yCcj-e#e_ExcweXngG58WXg$H&0eq4?L)4TkjgLx_~y2`K2 z>Lq6{dc@w^LA{3>V{6Jgj%~5iH6qH>sY)f4Fth^#G&QxLx$+%qKFGT ziW43!)!ryY<9PPMb~7BTw17MK?X#Q|90qgKf7tY%m)#}gW-Zxfg~6%$VF@%eyP|*6 zE`A`Q0xKd(epQG+)DC+wLTSv;d!pfxue8owYeCVi`j0~7ORw$@b3xu!;H+{tDJ-}bZ$Map*wSuKi#2K3M>rdyq%n8>0)PT*P6s$j$?V>Gm zr-<%Jh<^R=J}5+1fM&7-)b)$ld?(w=^7~b;nbAbnCy<<5aZ`pc0|j8Ks)nFoDI4qO z^tQE{9w})$z--9#nT&RKiL%oG4XS#KZdBRTq_6I|39IL6FHSpNjbKRn{1OkrQ}IC? zZ%762+54O}3=}FqmL)cp+IKo(pX^VH&K^D4hL0xsvE!_pTvAJ1KqiK>=CdQ02~X_o zSnvNI5C%sxd__B`T*E%| zD>H<`9NnMFG>R|XpDjC6Om_4xn)ce8Zc-h^G8yl*QCkFBnqmNh@>?S<=Ux`o zc1$1^Mf7)1G^%((xv~Il@9LBG~tn_jYj;O7HC*1?N@)8{NcTDX6lF4 zP_x&-?tpAu_oQne^P$!Z+?2gIuf;q%ZylnqdhL9hr-g!)#DtsVPZ9Q%K_#KM`(^IrQ-x=$O_CwRK zYE&$R{c5r2TmK+-z=(Zo2@$nHiAg=V$jc!Bs-8FH_{9@~(3`c!z|P9n9ky#TFgrQT zo%Y3CH8!vE#?nuaiVoQ5qxrd{snlPb=r9#R9zbYaM7Y{Q4~rdMT{+bQDmu6i|iyMJEii?%pJO;`_;d{bLZUWob#OLoJ&DoUPVbsRMr7ahLg!e zv>kBEV=KJBC@ipzFF{^RI+}vjheU7{7HuMEJ>VWm$Py1zEBindQ%;AK@2V@74uO zoD!>S2`g$=74U-gigcd?ALN<1aB<6S;b;7PNA4R1h4jln`vY`eYr+%px>ZZ^nrn{L z?>`5;e59|vfc8p2f4(Poe;gm23I8>{QKY1c7praj2HGpr{rH$BsX>T44&IG&%Dizb zX-53$4QQ`I_a&nInKm7QW|>04vpL^dzh7Rvm;voo>E2<}2kC1L!Qp`$^De8lRAs!l z_(}@e)AXmA;+)nOdlVu!|KTR3DY)3{28Tmw1xEC58}Mx@9t*iA`Y_jg_HIt^v+1$@ z&>o0>o4>g{#zZsL)_5^LYVmSrM(&pb&>o0>Ac8&LlAB%I+(Rb%e)-&xOrg_NXb(ic zTMkw8yeTDKt?}0nd7O7^T91u7v~zMK z{!*QccQ1qk6^X5FOGXQ)pgldB-&BZ%IH$V?Vda@R7WdQpj~;per4<=|bc$OnRmXpx z-O^;=?DhYoYziD2pgj;x&!0-$Ol(o*6#cn1<+UnRg(cS%pgj=H%nq@87mSR|bmu44 zH(YWLe$KLI3EBhEq}0z)=FT8kJCy?7X>s5v=d$cXpgj=HVaXlD_@et?IscP+Afs)z zVC&)O0`2KqF=XSWqoK#~Q$vZYczaFP*`Vi8`T!$3KC}Aok3P$grNGyc1|r(EbW=LM zLVMu(hQ<9=d7apjtCk~!@u_jw)7=ckp*;|trY-(x)18`cc-w@-&rGs>Q#y5T53~oO zlVhl%muwfaaJrI^5p`klO7XCM6SN1S<1^XZ(8%)3UW|RFI^Q{H?6t+$Bxn!Jwe=6A zWqy2}x*-=!eFsi#AD9|2Q>2m1x0u_or!2t&Dk;e7o>OjavL;Xb+6mi$?i=thNYY{_V58Uw_PY zs%gb>LwjJfv<_*VeYC4}u31DVIiZ=NclQM;kEEZY`)qVz{jFwKw_KrZ ztk9mOClwFE$+*GRsV7ap)RaqWDPItfd1^WHcnt^=!d#c;=0wQRw>@p@s=p@Qf%ZW3 z)rCA+N2X!n+?Qkao%+3BwwrwyERbcC2pU%(5U$H9^_~#Gomm5Q-%M%7d$dKKD0PA*nXyVckVrC4@AFt*s6g0 zm6}!7;IrCW!6mifdIC1ko~ES~ckj6_JZGgfG~2R8LU`lMlMu+ftip)qF~n1Z-gMRW zGj&7k^KWb{aT^b`L3<#Y=UE?Hy1}{MhMk($@XF7uo~SY!*Z9RIt)W*S#>E#&@uIQ}M%G zQ)mxFGyZ0ggIXGIL)zAS+pVTLWhX=z6`?&4AH-PiQggM+QLaa8RTGLu1YCGg5EZ8i zqXO&lo!&kNdbt7%K5rN=H{?^QbZ(FNNF^<6^ zd>pWD-RQ0(&Nxm54`Qn>zx||4msC$Ra#P76P#Dfe*fy~gTyk3tuY z3#MXjt3FY>2a$P}^GN1;9^)XE&wU)I-u&+u)&$m8NclnO#^j>$l2G&xDJ#je;Y6m-7EhygLdL!xY5&2fBVBh1G zA5eO^!Wd8=xDb(Iu64llaF$~J^$o2NCJ<){m6ZzPjYtWv6+#Oil=(m=AOw-fzMuk8 z@2IR)7{*fn6_2w0_|{D-NU7d08@)a{4KkX`6$W>v-Y=qe=WJDH;>90glE#WJ@DL}z z$8_;D8fn1NNZ2EN?e5v>C+?izbt)XJ->Q+1`f$iaB7#Zn?2B2?fxx|@2cf6loVp~ww4aMfXWlLCw zSOg4%CXoTS_Q!pQ~zV=Gx&gSNv729k-QpMh4Ryy(kElLtmlNS3h+`^18&0}_WM!pTS&ffiPJ z6CR2Gyw%PPy_*bH4tlnWUD_5l*9mR{4%Z!*pQF54rAW5$GW_7TsprmUSn;yO+uPys zPB1qTM%JE0V(HxxwoaqfY-)Ra>cda=)X>xI zx?AHV=9nMT+dnsE>}%vG3d${$16u+C1H3oMjNIDo9l{;qRp)9lBb2`MV51Ef673z) zSOT614FCI-{;_o&m&eE4KRP~NN)T5#8D_m&y8u17qkF1*GrP#(&zdIx+j-uT)p}h3 zKS030;~;8>CL!&ChuVKhl6hvV~Pd^fR8*>3EdW8!tDL4cMcxKj@fyb5Vb`nEv zuuIqAS0&$VWUUrhwbC%4Airbwt`b!DzR8bZz2m zqyh6{QRO1>vilwX`HtJ~lyWEdKU zK)TUe*tH!~DU5HuydRgWWR73Zm*#%v0nq~3>kJi!?`$7F$Fi#qJ-xqr*yg=i46`K? z-C+c@y%Vt35nH(Sj#9X!O-Y@Vekk8iZF&8FIB*mi|D6>UW$mx-Jojp8Pqv>#QQR)D zM3eD&3~*J(nInxUQ65;V7uqZ^=j3s%)@U9qF-SNL7?&Wm+^z@b<zT7XOCq4@e3RWWjW5y&Tk(&MJ*zKHq z-)=E}r_NxDC#O`tO`DF?npdyF94>?lAuHebx(imf{aras2>GB*~tL zwgcjvtI$zo;4Yl^%c|?FY%s}5*`jI_*wXhE>cRp7Vl<)bHd+>T&>{5tI>d?Ll1#&> zo2&KTIe#;@=}y#@u_(##wGpb;vtL_f4NX_;l>wiso7~IcLzXa%s6#HKo!#pR50+Ym z>2A||UUBX{uZe28{fYdwZ{O8%i3YB6E3-HvrA)}sPt+&zaK%JSj@-GY>=(fUj)IfO zFkl4fJX>7y*QS3gq@2B`I-RASJoR3!>F|TVNvrD}PO+>>+cD~)g6||=iW-6; z8bd}B!AVqchvT=2f2VBrhA6 z^?0Y6l|E`eICqRkJFTKJ8{{19>8dj$I8=&Is&r;*y&~To3w6D3!)0KFq-Gg7aXI0j zft}rdN^4N!0KT-iDJV+#{>Y!&&aJuNMnr1uN6)opm96+LQ^C-J?@C2WQdjOR?jnL6 z1x;rSFCz<22j%VCmtD!0%pL3Z!Rxy+h&jSZj*JKZb9#nz@mdv*JAd%V#H&1S-PkG1 z-(Zf!0KeW|^wv0Z880SBBEwygz%#`@?M^nfvpT9H zaMvP;&%4y`Lqin~f0!7FHq$AL2-r%8Sf3i6 zN5m9{qQo`_uaP@;`~X;!KuL3*2dl_)_NZ?*z2lainDbwL?R!Cv=%_s=dzr2q7 zG5SEWLvT-cNRP7=zo@$hGhWG9 zw-7!pEb9paGxw{fv78%xkDFg4wi^G#sU^nWALzUhk(tBE;b6qqwpS5P@NHLU*)4|-hQLQq-_Y5NY0 zda48Fo!cneL{@;Y!_SPfGbXQjnqDahJcDWhq|9%ajpm5L36%1G-f7ngO#*Utu7I%< z84gU-n>D<;nsp?xj0-%e+#7QBZCv*9GsERqyV|elKy24V5Q0nDT={#4WV38|!P14U zV$5y#T~I{9Uwc(13&@(f7N$t^fE12)P$nV)d--u{Uhs&Zuti6>#V*#L##6|QPLQQ7 zc&5BAd;HFQZvODxSXmtW;Dm=h^DB)#`Y;PKC=FymYHifuo0u3bWGFP~7aaI-c}>E)$m%iWIJva`v?kVRDQ#Z_!k2eIjOGCf-+n z#wGM(zl(NY1;{y5sMQ6i#QDhPHOO~l^VdqQn)}1~Mz*{i-*$sI5eY~9T}#fkKlxSq zvt>KF^BdR3?bdrOZEV7XS5iiYuyfNf{Q>ixQLh^ADP1hG;8?H7B<+#Zle&%7=AIxf!t*t<_B2nSvyN23hcc&F3s> z0;ctv#EoFqpq<(#2yLzTGg^;NSq0}0Jl^$d;e72X&>1|w|B~p4}ES<6<2H+4q0XCYYqt7T1yEr z_n3MOHiPJync7_mS0de+F_JOIF7i^YY~pPnN>N~4uQBgSAtPF4!4kMsv0M}(OY&jd zx9yn!@^q2ad*{%}0Fa`UCNLyz8>8!%r*{g9THd#~vOVvMJQ4TJ=hw)=)p}*r)FXHM z$?UgkN!^~>sruO4=|V~F7BF_CdNKv5z_w2sUKoxxVm&e&y+z>w`*g{}H8M;V$rvjh zjW*u;BoXAJwhhUnE||WQOKu8Gkh;W{C#{aY#y#r~v!9x5H1sCM%VB|h^Q){YxZV#M zGeeAhG?+{E!^Zenz>0SU9NVy(|J_0RB`q_4D{z*gYjZT7>^T%3v!`o*wO%gtXLO=F z9qsFO@?J2Il#^MT)V@(uYL%tCWBGi2Stf4>@bF7V+7i9qg!^6)+2YB!n!G zxVV0v%J~zlElD>z*A&_^Lq=ZOfrytIbV(`B-2dxkg!+!2(q`><%#ch0vcCTukJ5(Y zF-IQvRAv77$ld(4P|b-+643+_b!>qSP}oZbU^2~{%EW$MdGr{iE$nFuJ`9j3c33=u z!YCcQS~Yr&o`p3ROYIOdoilK3j2s2;J|q|5cr6yM7AMlU-+kh_+e!JJ+*kJn-Vngd z9czcjpzT+pYJbP<`i6ts-m-fsUR8HDRlbt>b3e#0@9aNGid)lJ&v(74Kz*&Yd)V+L zKTd;*5&x#zPqnxWnJ_qb1r0V3id^>eZ#E;J-$%ge>r%MS7~IsV!l~O@4cA}s2-+?4-{~mP^I{~RYEt}E0vUTk)VIEHja2c!emQgz>htTev(xeQoJ~Sa zXqA2GI&;{6HE>A+TPb$gtlq8(6MNE!in%+vmST5`b!OuYrdAj{b>9MCg1%^lQv%ry zIZ?kuIpXXNWthB@Vf(kmO2#N0cvyErFsYSY>AZf}zEaYg_FW?&?g9sHZ2D$OZV6W# z)|kNuSeA~ove>=%7y>gS5sh;H`_u}!fMR3wnr80;7OtMLLkCr^a$NbBFHaZ$FPm7E zuCA-R!E%zFosu4U?axMVAx%B~z^F+B@5;!>`-10{_r+{U=ZozN6ImEitC`ygzUFmx zq*Xs~oCBjksp>Aibwm~QQ7^Kh9UIhX$!#PHzQT1SqRF5tx+3V|=^NesdG8G$#37&8yYH3H?H}YS1hzcHFbXDTh8S4gZG4^;OwTBcko`{A6Dft^{1kKT?d;QkG0^MvLG`0IO~n>|iYi&hnQi#fTMN3Ua>Wy!sCAz! z&~Is0kwg3@TOusgG5`H>^&bFP++z3FCOgYo zeo9Wu$rh|QgnG1Ag7M2}1vOwQ5kgX`6yCht2*-$xyFKmMOf z>uETmgwX+7L4<_^rHf?P+Ao};$rD8yh8rK8INtcD_n`X zb+EymQmdb>L9*-Y&Ch6E>s@8(#XQX>B7VYc!|RPC6WF<09Sm~Vuf+e1Djcw@jZo8}6**-6U|5Ie z2^RA@;e=(e%$VAvV6Hk;98(%oNZEVZ__Xk{+#<2M^-g;G$(ATkf5Xz>Y7oh zDiWZ4VK61}WwjL2VJA@w<-cR+g)xawanAR=o$umBn8_;Cdw|;{j&3)fd9wcaY4c&n zvuPJy$l7l zVbzo+>ma#};ZBmBYjT0(Pqwkoa7IVNa>a150H^)(lE41yGjJTs-8_o4CN3*MN8~?@ zgrY)`nY`ol=6)K&-i9oSv5{BlcjAYtJoQdEXRe88KwZdW$Awh&$?YUbr}@8gp?rTB zT%OHC3FL&H4tW@a4<1{X$!G%S`LqfN(1VYUY5>2%*7xpafq%z522YQnaALM*R9qR* zlgKx-OZUSrVT57m;al6yM%7i+RGMll*s=9ek^Qc0MEqp}*#OJwHM-B)Z0m6QpBr3u zwtfS^AGUbTwz2|voO8)fF3nBND@iQL%+J#gO3m@hFG(%dHPkcHGXcVqiV^_a=nKlT zwgz~d&0Fhp+qe<`Ed3R*ZsJff<=Af8Tu+u~oSbt_FNtpwr)isJIFtxktSM6A!?v!; z|K5E7AV5-%bJKQu9~=>n#bUAh+Xryp#pyh$A}I#5Do)Bc-Cqv+-{hsOth{KLmdYY6T$ zTPzCPUm{a80NR3*c2lin6wEfIEKDctBY69#)8N^~U&Xw-mBv z0EcGb>nx5$7)5ul!~goi-n=i`dnaBx_5!~_R`3_}_;wy9NjCQmeVURjyw-%_kVlVC z6oIKQ0;gu22#qZ6Uc3+}(1@f6JwcdDqWP3ga&H$b+c#TOBVk@|C08rJ zK~HxzI3paPf&2}pd;nix`n@Qlg$iUBz$K@?Tf{Q&z#;?F{l=EW*bqgwaV7}pfElNz z9caUitF6m0Pd4K0{P}<;VIv}Tnt21lPC{>yA$qKG|xLEzg6^KT_Vv3)6cz?21QO z?)Zy$(aZtTT*Z!siR7Qb;XqH8g6~-5*$ViUkjwBeZz#eAMmLbofEmXNioKTLbLt=* zKF+V&0S^Z4GN2?<2do{DDyhyrL}o{)0Rv5E%#9>f#KR0wM2MM{r)KIn)fc7#HO3d^ zW-ZfjMFYnt_%-B!atIN@f$@q9038lMX#rs(({}h-$DSMs11wz@;`7uwPb)AzEh&Tq zL2m?r7rIu8F!O8xek*!Vrmcx6!Eo8xPwIjN&^1JQ3dpIjzYCn zz^6edZ(joq1O_Ic+9545!SNyD1_iXWM0Xv5T7wVtV5E*d?)>SWgVWc4L)U0d8QbZ! z8@UGct>R)8mhwtYen(Jt_o;+4%0QKpFcAd ziirC?mtWA93V;k#YSd2{$HDGhs%J|I+$Zl43ncou=zM@PY9oIgXl<#pk24Cf<5ZpM zKwBV9!eR}RvGu^GIH5@b!bzobiP34?*H?_I3u+V))Z1%Bpq3%ww~IsZM1^WGr8FW2 z1hAk}#Cv-xgkwb{wEYl&+*i8Vb5y=Dz}H%+WI&Q{C;9bO9Dgp;YNbRIkS>tnkgKkZ zr_)K-wHwAQh!oR&TrRaO68IySAg?}T5I2ych%5XTTOocZ5j>Cr97$a63-Qh={P>I} z46lW9OOU8U<|yNIj0GN_a4His$vOceLWVv-y=|W4HwgoH*&48rF*E1&VK$ux5oq`a zVfruoNx$4}hlDA*>Q^TnMo{F&8gkP}@yt8sBgMcQA4ObP&BjTlpgxu-ABpYpyG zc>MC(9O!LeH)l2QV|=2D)LRh`rDgbzQ}Ssn9GmH{%G;8x z6Vfr*OGrR4C7}@aw~F}Q!ejo{VP42~JmeD91egaRvrEhxacYO_3Wy-sWm)srUb2I~ zrVB!sxkb(_#ug?5R%gkz z=g)msf{s*p>Q1ohygg#OPI0Nq(%uOMfwpPs99e4x3(F{@stH*6wJ+)f+$s!Fo==~s zC~;azUECoa9KKrCuCCP5ELGJFP_%sGb)RgxC~>TNn_vqi_+avjDYwCYHWQlypDs&Mbk+4JDXvzHgAZ2G#gX5k7IP>&9V zrmTF#73P{P+ag@K(Q?`>APe28Cn>0@{R_7$TtRVbt4A7G_O+rRQ|6<9VlQ8`ZTs^!_Z<7*M&vV<3#Ryv^VHg7i`*fZ2)=4e*##@nmRS2dZ%pw8FvQ)x#ozXa$pbE(POEC(!TMkGmmjM~yAnY>4Ay zU_|WcZ9Op;jqaQdXD!A}*;7_y(}p9&t=nxU-HyNCEr^QhLo?Ect-|LM9NUt6mJS z)f8ZzZi{CUU-U4(MF?>FJm5b9Y-W(}+%SCu#3`3|5A44S@HNdoor0Ls^Qta~1X(e= z`(5LcO$bBr5D8x0lh#=my*QvX#{?_(DPRU|f)=}&X?a1DgWLC}*0pK-*3?MpOIHY` z!qi5#d573JdhL98q(UU{k6tsrG2g{9UbHX4G?aEk>EwVqBP~3nF0h=oo~rfR5N-pw zfp1lS2Y@~~5s!xe)98)Vts%qq@9AbAHvvtLCw<#h9o&Aeo}t{^0}}1MshdqKY+eQr% zwUunkB||taQ|rf!EUptG{eTosQU5XpKRpC#EWyArX+Hd|qudrAKoxT{1)4@>3WjYSk8G}=6qL8VJ zz^9sk-pIF1&+F+5jJ}b8x-qNCsHf9kqryjN6S6FA>X7S#EJd5dKPQm`#uAcs7_Jx% zO<9P?HBbqrEz2xTyaqBg#%nauqhX~os}kG+8qs5ju|S6pRbfJD!5k6qq5qIGyBg`} z8UkiE7g0#sT4at(bg%Fg()j3ZYS0AM7;ZZmdiL+x{||u5{rXfocuz%zhv4Aw} zAly#JcpHMNRs13Y5O{xIyb3onDT-_bQ^F0$-rdyIQ#X3W`+{B>&dXcE;@a2Y=Y600 znsLwLGz9T+2&`zTSB7{|+Sj#EeuAPpbuC+$nAE$O`dR{sQk#gtA7i>qV-xScfc#-Twwc*f=1)`VM%Uy*>R`+eWrO+keG`o)DV=G2L$aHcog; z!6cj#%(J1{Hk+d(Y#D1~OI}Ilqp$q$?|#guG%_abzCGK3MtAPa+_~R#HGC5%quFvC ziH&JdEOw_G&2MzeA{>3emS&t3VjLD>kVT^?zKO?+NpZW9nr#?eGv4tv4j<=`~j4M7$L{dtLPw;XjD2(K4IO zyUlN+WE@WrUxEnF`i!r4F#IJrx;%e>aCUUsv%7N70D!~gtk4LW-#|;8MB;qV`#JdW z@;Dft{Mr-Wb^i9fel<8ad)Euz_YRJF1AhINPO~|hhM6FoQI@4yUSywm3WGtP!9K%+ zt)&rcp3jgR@#NlJ-6&0R070Bv2s4T@oMb!^rzbxW-k*De;Yt52IJ^8H_TP%mUFV0+ z-#UM(L+~ZKx2g09n*XYuN4N-!sm4?b6(^HaFJ_h|*Sy?WdK=`6C>jT|cpew-ntfnJ zJda!`O}Ys%Q<4VJBERvHjev;d7Fl|CZ(!){GLNz#3Fpza{CyGTIdkMH+U)(%=`>$$ ziOW19HtL^W1V8nDIX@TvcNp4K8#q zW^8Bmilu1Pw`n$Z$Xj_!A{b{V+zJl(dpS}}Jd1(qTZ=5bo`=CS3dd2_T>%KYR)Q@7 zyV?V~rO-<@Leny$j@T{!EeZ-T$>1@#T23ZW*1AdKF-)e&wuO8h%?kWYKPU5Oezh$a zkb4E~=KnSu@;&|lrjy;o7I)~vd!8P5QC$I2;(!S?VB1^NYhSkR?8DF9S(Jz!-LgpY zq}^;Zb$8m_@}hOXTB2v!XIj`iB>% zP28?H1x+U6bX3d~ZCP_p!(==Ia*`;{$BL4pSsW$Bf+<-;DG7rU$#3IgG!-qt&=$~T z6iS(b8GJYj&id!Cfkbk;+};cNKVjDr=%6>eI2c?EKqF%pXXPMn-8wW<`Fn=BJp1Xa z|M3ivL+1w|la?aMaJN}pM2ae`b^h~w2S0Gfy?n>-f^a-;aqG6wQ$cocZ1l^Q-6|uv zOG}os+1D+MXF}_$9kjB<2dx4K323-Dp9WIoQT|)2R0ICoe$1U9#9#_WQJx2+j^^-N zcpc@WGHAlc(aCbI-x%80jtd~UB@ zXOU<*IG()~XP2j^Bp;+&?6*7na~SmPTXuzp?}DTLS&u;*+Dc|o4inImQ9W^IcKIEO zM}yY$R(NzQyp%H|7&#-cxruKC{}12n?BfXd-W@3l^ZS4`2C6(g6!%G@JDvdPO=|$T zj)q0UESje%y4T-SnBCX)Ols!qkDwIlYUW?~rK($TNX+g(!g$ZaF z9OVx%psVJLGe90xx+r4M1ufgZx(Q+GZo_ODX=8B30tMUp3`A%&U$i)k0Ox`o-VFc^ zvo>#Z)fv8yivX}s;_KilO=m3{j)rJk3cW)U_YZ?Ej2guBu<$~MIJs_ZDu0Jy%^>S| z+y0Tk*CEqr6|>!9!Sti&aEdi#6T1RQJ_?nm1-o_!uYBE zgt1o)iecBPF=bfxHwBW=DLW>1(0lb zF~9`I$=S(8R|?KE9qN*t=qG@_gRbM{qIDgS0Jb*HrR=;G8*QSoaWN;Y&E*2@*7N-J zbCd)tm^K4+)kcyBkVgd=krppvTgR#(bA$7bM{L~W?15xOIGXYpLTi%8fT(|v5U%p4xnYQHP*Y2v_4#T-=LvLi-B(FME-Tk+mVXzfYg-9?cd8y8x^xJiyj80ukZwoH}|B0 z4-d`{F5b%(Q}>bsXq>nw;ypP#?kl|59k=JvZpQ~Frx06V-khbRX+qz!+p{k^5Q=)U z#_(!QJ#yOr7z{xukAl;a4<{GW82QFqPr(>`7bhQjesnlRDTr%Oq<;!HPy2@lC|WjA zK=Ve)O>5)b$wi>}T63M_fVg zeE2i#V+)z_#n!@gKIs1ggs9xrN`0zCK*{scnu(2*9ob@b0nHgdg)rLIC zv-k@hw?QbB0)jl9N5vHU)Qw#sK2Bq>4&ZbCH}%y(Z_xGFfBgpa*8bn$Ow+vBf3>^2 ziz$REoC6@zO#BNhPYKXK&`_^vc`x6Jzrb?Bf)_fPW-TPfi@(1hDM$L0-n^zaL@fO# zS-t^FqIn zZZ;}U*WKV*@ZA5y6R{TD;sF&xClYvMKm^+sx_@b0d|Mj#p)6`3y1@iotTz3P;&3 z9c%NEL$hIG%a)x(lOc^(x7V}uDxC40NQ1vS?f-ai8XQOnCf6IXY?R8GiuSJsXj379RNR^cj^1PVJKnB@Hea^l?hMNdZ zQ!i&&EVK9i0<6b}gMS2r-v31w$z<*ZFuq;a)}ZKLPqlq$zC+EmR| z0Le8#w5MIW3kia0CTv=#F~KhiTD>R3M;iMKKtYmqW~UL*O$3fdFVW(6H&3`%uQO5H+{aIy&n&7X z->(Fsvu7)kICLI{XJKU#McE1{rtA-n&^x{89bWW~Hrl4kvqct7;ycV%x+8CM=~Ygy z<9eBG&xwAi2&!QhJ~7)l$r~eLj?o;=qA&rAqH{Zs9gO;q6Pv!%L??ymMTqBwl|w^y zW@wHc6|x%MeByDU(lmC|p~z4=cf`ZDNft%s7*Ucy&K2HQh@ri~^>)k|-rhqi(NeQ+ zYt=S_xwS^y7f~?ANnwMI3k<+Fbe#iSZP^Qu>+*Ik`vWzOghJtq1KGSHxn;FayBhBK zydvd%mr_nOSyS+N!NydfAVS?nr3y~2CwodmGb?yAtG-;Up4V#mR+-m&F+o|j54HcG zus$M^w_N{FYYAjAMUjO`K8dni9-n*+!A9bNc}U`h8O;w&0i=Xl}elh5FoXi z-U^7ODS&W%NJ(TXD#N;&bkb7fXMU3pQW0u~GVMILs)o|dQmrakTUiF;pP*N*Y1VM- z9xU?6p>21(rZB{rsEydxv8wgKBBUZl#VXWk*@i?w*!V5o>c}|@>BbAvP?$iWhKdGR%{Y&=%R8F z^tPxd^tCWhiucFM<6}hOluoaW;_z~K(f=SFNF7?Q*B9#rk1x*-Ij8!ht{(>{7rn3S zdwG7^KRA-dPHR*8UGG9=X;tAKu{`h;mFyN_Ry?g3a4f_EO4^HL$g7e&Hl7=@nX?EaF4At4QcHE83=Uigk|3D8e&1OpXdzdo-r*8gN(TMc21IN-LP7)`$8E zM%v3sJOl4w3Q;aDojo6vYDwO-aE+J@oZC)Zu+|Kna$8)^wm zVztcJTm@Vj+IBhI!EgPRtX(tPty!FW!A$z5nG;$sE?|UqxqNCu5NN_5s>!i$%Kk7? zv^g5PYSVT{4-9Bb_K+rF30eN9MMGuB=8ea%so5#=H^G}~S?ecCF(ri6%*!Kdi=xO) z_^<0^~7FXR(YlqwUHbL%eD||FQ^=;trJ|i23Pqu4}MRIv(21l zE}})vc6roOjoPC^@l-pOer@=upZOrH@?L{ug?5w4p2vJsnpnMa+NiQ@xz>|Nl`WG$ z%X-jfJhKqM7iliFullqnixO&vrpXvhp1_jS$=GV3@^hypx*PGc`XFp#g(jHG!n1%p znnk06Qd{Y=kieEt#EH*Qy-c*QMYdH4;7>QCr&U)r$U&`I=8r9w(Ps4#b&wYEg0g@#KP!0S@~| zJ>*~l;9M*isdm?ib+uBnLh9C*`>w6>q-r8%pBGEdPBz89k~K_?g`ON5}5 zGRnyiXq-bcu+&%|y1Ox{o6oghm1U?RSKVq4y;cpcTeKH12FpaKFc$Mi;R3ATB8ydF zp2e6>uxruD^Sa!HJR{o?WTyB}xr2^N=ysN3#Wb z9q3)oQ;>~&42eKk%^4;zDivVp!rU2tn@ZkQ6peEjIW3|VeT_8LwYja{r>$<_Su8xHTrY2sB8^Gn3(89v8UF-mOr49+SP=I1_8yC}nieQw&vA+LvizHNi zlBv+fvLgah;W$hWp24w*5+oM=4*z`8jWCzb$a0cc9#k7tzw}W0l$drHIDCSWc8WQ! zz|6Sw1SnfN59?f zcB4H7R+X7@pHn+D?;D^vK$k#@sFP>Xe%4mp?_JjMLg z^WCld6%ggyZIlLGzg06-!-I)FAAr6uzh!#x>@oLRrAB$>wF_Ro+{c-(c0V>1U1-kR zHG_25+rTwh3l;UWB=$ES?q5a-Y0dg`L4m&1$kgwSf%weeco1_euZJACqV(a z{9vBrP>rLp=Fo0VoI=xv{F>YZvNVLQ%=W6W#%R|_TGWI>-m^o|tA_|5`-l@*4@hUF>P zS4?k}z?-8ZvJt39xlh$hl)MnzUf4E5qIOIU^;aD{rfgeds7?oB^k}usM+mNdF3&|9 zijq+&<_qjpF)XMF)ym2)&!+4|-pXzf%@?q9YG^k0GNzqi8AD5g@3LfursiL%Wj~)# z2?TF4O_3PH!c}Gz#e0qG6h8^$q=@7g23vC{*kqV}XiO+|QT{%wm01=9#0;Yd%HAMF zf+FSpau%tY&iQqfTO}6Dld3Q*)33rFa^r~!O8ymoF)G3trZw(m!R@#zbGtfe zfD?Y^k;=B&z2~`1>xU!EzhglBJjc5l+|z91O4qGs5*4E<22q{>N*Ah6OaP@<>~ov= zZWyoUB)>InHaJb@#%??uJnih;)%0ckKy=yy*ae z`Cr*T;F)xEs@SVl<$9>#ORDqICYKT^wAIKdc!i#uJ1O{OfNz308N&~zwly7<7N3yf$zadm-Qr2BQf z;|S5!J=CpY9Rk}ytJ#dQW&k7UCPQ}s!QG8TLVo(T^Z6^J&I(_Y+~G<|?(&T_ zh0XGGd?u=FpSo4k&Z6b2B)+*!yj3w$S10CU@kYpN`lNxAx7cBaPqF-*fbOSzasJ{p zg}w4j76*>ex_paiYIWD2xkt+~Uo`ql5o2UWyrjMsr~`!mzWW}acM%%DB2w|SV@EA$ z0M-T+)NH+=TS84eWNa%Fi}udq67i$ZzwUMI3KAJ;TX*ewXo=3Gq#)&>|Cw-pUJ(y5 z@UIUQmF8Dt&RHiHf#af-J1qIU@&a$MjQ*`N`uMQ3j{xM8JI7nnO?E+u5T#zGWQAXN zBIV%|uUr!Bmy1NFH~69U04^n4_jaf`&09mbuBW&>tyIIkQzuBVQX*uXh1`{?4O#f^anEi3#t@y?H5W%zgXx6IC z;^hF9ajta@>`L;Hf}XrD~9!yG7{JG2Qh|n0LQ=R zcCS$W$!;tDq)P*BmtRSkcKshUu??*btSCZ-vtC!1oLpxhoOL2Sc~?642GO+K+*T(; zwiK06XnAumsLVlavZf;ZoU|$7w9_^v4YF?bMh7Rf32ZLiIqFa-yh=0m^@5|8eWx2f zCFM@hm!7qI`A!U~YVrvvO4aZt=l=v8E0fFh-K-QdP}B2inXdFcnaCpVI44&Fmz+mv z+_H2o=We+s@e1G8bk=(mglulbl1BArZR-M6iSYD|{0SPZw{F{QGe zz_%Rvk~6e1z)4{EOAZ^}NF}3Rywd>>OvLk_KTHF z8LDelPt%(>>QZakX5^m&LXG)ucoW8Wk%my0ox88mDdM*$Y>Qum;V);0!SU(k@IBiAfE~U+*b6QfV+($?O>>a4 zYz+p)^iRquH20GR;w*cfZ(%kN0C(5AAAAMZXot^z;m=oilS7uT#~I5?rm{9siK+?P zKH&`!c%i`OuIo0EkN^;|YbAw2nhi^#oIm+N!Y7%ZATrL_=xX+_^g8Vd2!Cw$m<%hv z_qd-E<$g}Vcf|X8Bok1kOcAjpy81|GE~elQjnd_8ET-X2BqpFJKr+Tla%-{hcVia*b}z;;D2F1=?q%&~nrw6lK>1drby2)cuCl+G7v0zQ`dI?hL#H++L1&?QMxA@$Dsc%K|+J4lETF~ul!lUQkI z|k=vSv@R%Lons=q9Am6sOR_sPV~K1f6DfO`P!i>gXX;$fbv-{k_aE?H?$`DC<#XNF zeSIeZauX33uCN_rZe17YmJ9ZnOQR-b(jso>X11y44A-K%_Hg1{6jtvnnf)OB_SI0Q z^F9{l-NM4x6^U=;;(DcpY7#*4ss{3=K7T+s7{dY}wb0d-&wZE3_YETxlzS^(hUsivNdT7$~Joa*> z)A6cH#~t^!LNMuK%(jM$k?9k=&d8y|(U-(`OjY93gvyJyT?2Fb$ETNHjFvb2YzcsK2|_p zB&;RiqX#KF^O=da)bg<2dwu-=mOe)cJ4&!Te%y0+UgxDEB|{emxxeJNJ;JQCPTr5a zr;G_xGML)U3U=I}XUCSkAI=D%mgLRK%Ubcfr2lS12|S+=*1!mhkI5@m(RJ0sy3+SW zAt@Sr^^?>kK6cRJ>`2}%RIP5b)8xr*{y4A4iUdFM){1(iL2@_hoMAr_we&)I*T0T2 zD{yT&@GNn6C=Q=EUDMLvpff|UD9sPrp#{(zHmjOW5!d5veb{Z8_ZP=#{6VMq&8 zMy5)UHgzbM^eSd1WqrG&rbYOZlB9uf`~+1On7{AAW6QKNOl5cj-DJ`_e_>bWsiI3R z-clN>3X{rqPb@)=)Yr!6C0m)BC|J<%z7`pQ@wm6Keto ztR|MFU513+W7fAva?=LB-Q;(rPaNniQK*W{_;~roI_0~Zv+2SHC7|cTNL8kHW4mfp z_`N&#?Om7TS1vY}*-u*aGKp16p*Vl^WgHP585MRk zG6?48@`uuC!ABj%Z!xXa*U$Fhl>*RCtQn|?^iNO6 zUap46gh&##wwi*22C0ywcq|+U)N?#1StbkNhV|Tif7Kkx3Y|X#mP_0M;^F!4u85R5 z?_Tp6bNci#?_0~se8@yn69L2+H<8^B7ceq0J1y2O{$G`v0&1XY=?yL*OY@E?%SwK> z3P0#5fc2D&p?$zX^+y0WU|)I^#+>XFn^jm`c{V9=W%31Z64BA{)hY&b+$Y2nph zI#^*N8Z^c2f{RbnVMtK5QtLeQbvQ?!i1|fF2gV-cmX*ox?RL7{zWyGaXW}Rd^61t8OKB#`n(>qMRN|7|37fU))__V1*sGfed=b{l z&~zwt^!X@Cl^jb^ka=S^0Qu^va6wI&b6f!TeDkv&szV%3arIfB63)NUTjByJW4H?5 zIW7bgJ+ywS{?vV>?8+tB=moh*G2TKSh;}>{0Xm=D1wxBtBP`E9LMo{Ot?dR#*Q9*D zD9HB|91f-%V}V61WBkq39JXPYCnC@0FT8n7_#`-K><@;R8mV>3r=N__3>ar9;lywepf)gPvJ!EyF^r=|zP^i1*#JCC6!r2%Co1T?Hxi7!Y zJdswbYMBap-Q;zd4I%BGBLGCWPwCL_sve&UJn*N520AmeMhMUcM-&(E=?Pl4#8xh3 zlPoZ(d$^IBQoU9V#X0N4&HS%Q3a7ky(vq>Kk}WcKUrbQ0v5vzUi^0fq)>=9`=zyq5 z96p?Ih4^DoPAe96v+!i?*3w4J*uS1kSSPhsf#z<$fX|e7cCp~zr(ZgiJoyrz0l}xt z+rY3QbKk9%p5xogBQ_y-m<@eu+_zom({|WoUVfZQ?*u}7f5+vWhU9J-kh9d#qCuZF+<@B z2!`}|V>AHTPDsDyXyZw#vXYzcQR{|dv&u|P7#qI-D_`|ct!)l|JMd@3h(1n40tu_N zK~X1s{3X;026Nowi(H2%u$@SG-!u1>ZuA6awp+QE=X!S1u66V;*KT|trmL&9H>7<> z55grdLSGkn;Z0ijZr$5L?(Wj5CGm2gfx<%BemZz$^pMP-u2l)d- z?}!P9yxdA)7=xt3<>Q;a;_@8wCj7uwbYLdzAG*=Lr(r`ML49Sti|7jtAZ^5f;j0OJ z$_3cdWN)P37>?IMlQAp+X~^rqfelpcJXa*+c@fTgD&!qfVV9Y{1>qG;4+4BPwiyC) s8|!*~4Xa@^J8%hCPWstmhwg#c9Q>8h7EoFYxtCroOBv5K!MKF~2i=v?SO5S3 literal 0 HcmV?d00001 diff --git a/test/lib.py b/test/lib.py index 723958ca1..ef08c2032 100644 --- a/test/lib.py +++ b/test/lib.py @@ -10,6 +10,7 @@ from array import array from cStringIO import StringIO +import glob import unittest import tempfile import shutil @@ -46,10 +47,46 @@ def wrapper(self): return wrapper +def with_packs(func): + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + #} END decorators #{ Routines +def fixture_path(relapath=''): + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + +def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + + def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" diff --git a/test/test_pack.py b/test/test_pack.py new file mode 100644 index 000000000..0b9e448cb --- /dev/null +++ b/test/test_pack.py @@ -0,0 +1,16 @@ +"""Test everything about packs reading and writing""" + +from lib import ( + TestBase, + with_rw_directory, + with_packs + ) + + +class TestPack(TestBase): + + @with_rw_directory + @with_packs + def test_reading(self, pack_dir): + # initialze a pack file for reading + pass From 099ec0dbd23bf46cba7618768e628ceeac7c2e17 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 15:17:29 +0200 Subject: [PATCH 0012/3719] index reading from V2 index files implemeneted and tested. Added LazyMixin type from git-python --- fun.py | 1 + pack.py | 190 +++++++++++++++++++++++++++++++++++++++++++ stream.py | 4 +- test/db/lib.py | 4 +- test/db/test_pack.py | 2 +- test/lib.py | 2 +- test/test_pack.py | 38 +++++++-- util.py | 69 ++++++++++++++++ 8 files changed, 297 insertions(+), 13 deletions(-) diff --git a/fun.py b/fun.py index c766f8e09..883062eba 100644 --- a/fun.py +++ b/fun.py @@ -113,3 +113,4 @@ def stream_copy(read, write, size, chunk_size): #} END routines + diff --git a/pack.py b/pack.py index 676fa26c5..a175f48dc 100644 --- a/pack.py +++ b/pack.py @@ -1 +1,191 @@ """Contains PackIndex and PackFile implementations""" +from util import ( + LockedFD, + LazyMixin, + file_contents_ro, + unpack_from + ) + +from struct import ( + pack, + ) + +__all__ = ('PackIndex', 'Pack') + + +class PackIndex(LazyMixin): + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + + # used in v2 indices + _sha_list_offset = 8 + 1024 + + def __init__(self, indexpath): + super(PackIndex, self).__init__() + self._indexpath = indexpath + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._data[-40:-20] + elif attr == "_packfile_checksum": + self._packfile_checksum = self._data[-20:] + elif attr == "_data": + lfd = LockedFD(self._indexpath) + fd = lfd.open() + self._data = file_contents_ro(fd) + lfd.rollback() + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + self._version = (self._data[:4] == '\377tOc' and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", self._data, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + + + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha)""" + return unpack_from(">L20s", self._data, 1024 + i*24)[0] + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._data, 1024 + i*24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + i*24 + return self._data[base:base+20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._data[base:base+20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size * 20 + self._pack_offset = self._crc_list_offset + self.size * 4 + self._pack_64_offset = self._pack_offset + self.size * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._data + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i*4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + @property + def version(self): + return self._version + + @property + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + @property + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._data[-40:-20] + + @property + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._data[-20:] + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = ord(sha[0]) + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(sha, self.sha(mid)) + if c < 0: + hi = mid + elif not c: + return mid + else: + lo = mid + # END handle midpoint + # END bisect + return None + + #} END properties + + +class Pack(LazyMixin): + """A pack is a file written according to the Version 2 for git packs""" + diff --git a/stream.py b/stream.py index 10bc8901a..44c7b945a 100644 --- a/stream.py +++ b/stream.py @@ -361,7 +361,9 @@ def read(self, size=-1): if win_size < 8: self._cwe = self._cws + 8 # END adjust winsize - indata = self._m[self._cws:self._cwe] # another copy ... :( + + # takes a slice, but doesn't copy the data, it says ... + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) diff --git a/test/db/lib.py b/test/db/lib.py index 738eeb851..35823059b 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -1,7 +1,7 @@ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, - with_packs, + with_packs_rw, ZippedStoreShaWriter, TestBase ) @@ -20,7 +20,7 @@ from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' ) class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 29b348876..6faff4695 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -4,7 +4,7 @@ class TestPackDB(TestDBBase): @with_rw_directory - @with_packs + @with_packs_rw def test_writing(self, path): ldb = PackedDB(path) # TODO diff --git a/test/lib.py b/test/lib.py index ef08c2032..6b25876d6 100644 --- a/test/lib.py +++ b/test/lib.py @@ -47,7 +47,7 @@ def wrapper(self): return wrapper -def with_packs(func): +def with_packs_rw(func): """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" def wrapper(self, path): diff --git a/test/test_pack.py b/test/test_pack.py index 0b9e448cb..a0c8464e0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,16 +1,38 @@ """Test everything about packs reading and writing""" - from lib import ( TestBase, with_rw_directory, - with_packs + with_packs_rw, + fixture_path ) - +from gitdb.pack import ( + PackIndex + ) +import os + class TestPack(TestBase): - @with_rw_directory - @with_packs - def test_reading(self, pack_dir): - # initialze a pack file for reading - pass + def test_pack_index(self): + # read v2 index information + index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') + index = PackIndex(index_file) + + assert index.packfile_checksum != index.indexfile_checksum + assert index.version == 2 + assert index.size == 30 + + # get all data of all objects + for oidx in xrange(index.size): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + # END for each object index in indexfile + + diff --git a/util.py b/util.py index 291855630..5c2bb540b 100644 --- a/util.py +++ b/util.py @@ -1,7 +1,9 @@ import binascii import os +import mmap import sys import errno +import cStringIO try: import async.mod.zlib as zlib @@ -16,6 +18,22 @@ except ImportError: import sha +try: + from struct import unpack_from +except ImportError: + from struct import unpack, calcsize + __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset : offset + size]) + # END own unpack_from implementation + + #{ Globals # A pool distributing tasks, initially with zero threads, hence everything @@ -76,6 +94,28 @@ def stream_copy(source, destination, chunk_size=512*1024): # END reading output stream return br +def file_contents_ro(fd, stream=False, allow_mmap=True): + """:return: read-only contents of the file represented by the file descriptor fd + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except OSError: + pass + # END exception handling + + # read manully + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return cStringIO.StringIO(contents) + return contents + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: @@ -93,6 +133,35 @@ def to_bin_sha(sha): #{ Utilities +class LazyMixin(object): + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is reqeusted, it is simply + returned from our dict/slots. + """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + class FDStreamWrapper(object): """A simple wrapper providing the most basic functions on a file descriptor From 0717775fa51460335976a046072cf5c492af2a91 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 16:18:04 +0200 Subject: [PATCH 0013/3719] index: added tests for reading version 1 index files --- pack.py | 6 ++--- ...438c19fb16422b6bbcce24387b3264416d485b.idx | Bin 0 -> 2672 bytes ...38c19fb16422b6bbcce24387b3264416d485b.pack | Bin 0 -> 49113 bytes test/test_pack.py | 23 ++++++++++++------ 4 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx create mode 100644 test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack diff --git a/pack.py b/pack.py index a175f48dc..2ffc64f52 100644 --- a/pack.py +++ b/pack.py @@ -66,8 +66,8 @@ def _set_cache_(self, attr): #{ Access V1 def _entry_v1(self, i): - """:return: tuple(offset, binsha)""" - return unpack_from(">L20s", self._data, 1024 + i*24)[0] + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._data, 1024 + i*24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" @@ -75,7 +75,7 @@ def _offset_v1(self, i): def _sha_v1(self, i): """see ``_sha_v2``""" - base = 1024 + i*24 + base = 1024 + (i*24)+4 return self._data[base:base+20] def _crc_v1(self, i): diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx new file mode 100644 index 0000000000000000000000000000000000000000..87c635f48cbdd3ef50ed7f14bf339382f9af6e5a GIT binary patch literal 2672 zcmbW(2{hE(9{}(xgBd(K*&dTMYnGWTWyziz99xMPjlC>~p})toMV67`B}1}TmM2~q zOyNC)B$V=>Y~wFNCXZJ=WU2ml&dE8R)9e5G&pDrSzu$ZB%$eWs-0!^?fWLg$V76@! zjQ4fx<(7XHr!VEZiu;rwkvFy2WR<}9M0 ziNW?u#9{x}NWk%Jk}&=fDcHZ8H0;~0S!1ubC&}!{~9egzD*m(EIK>r!k7i~*+JOuqz7{r{l8%V`*$;h zeTQ~A4D+ushU0(71dg$o?qUY>ZRRlkf+Roov-HpfiknPsQSgc&e(qq6WB`1JWKPan zB@S2bs@9j0Pu|~d3zptAo6i9t?h^kI1DSrnq$x0?CZnAHC2c>5vI8K!Ubt5DVi7iY z?A_8z*$9G-R#3e*0)XYZY(?$$*T}c}X^F@3#WAThDSD(2@KOCVt`3W24I`2RsnLBe zS3jw0nq2P#z07L^-wp$TW4myC&t@7Gr+ZVVr6g z$-e-=Eo<#hN2Q21t3Ra-jQb!Z=9wbXoA5o|tkXGKDP`nmyOsVhEBFSzNeHEyv<^U> z48}jq^E#hMi98}g93TD4@T8`>B>;(nSoROf%FTl_#jluKzhpcYt}lD|3V>@9rjgfirN;=VrfBvag&LO4B?q530k~0*+caMbZa7Y`tgx)%5tzpQd;u;Cr29|` zg4`9@hf42=_8lvAG#>GT`&3S$x6E;z*~pNXEd#Z#RO_xjl{W4P*G`Rh8+*|#F9jD) zzzDbvYWkWWLN)=2+6tU8v~Iol{rb;BG-u|EgvR$DD^I}Xt$J>yH>2(npZ;`;ap+T_ znF0E536B7_Gth=y2XN&VQmRuEmkqQ=~}@d@(%C z18d}t(ht1#T;EwyGmkzOP?_LDh^KKd&j66;5L;?~8AqfzuAk{?`@lec{4O#9*1v>p znw+nYlJ7^=)DV*V%wjyr>I#OB1M8oyQMA%DR=eLa4S{}gSy0&{e^`fw-dVq=QcpE!H4pW15dKd_d;q9mTz)fwH2@cAKACp2 z(fn9~Q7%3ZfN13_?p7)l=hMSHgo$4NJ~#DHajXZ@8?8!Uy7Gtdti%pQ4XVV&guHY^Hum*BmVAK=#l8yatThW*Cw-l zA0*q7;d9|z3{TR1LV}0_<6emXY1JSW9tQuv_t39)r0UeSghr*wy)K~ytmL%wom}|) zzBI=lB7L31T2D1nQDEV>+91(#F93cD>|VO{CQDY@>+jHY6ZJI9(SGOU(!DkXfXl`HYn@z}jw8u= z{7#8S{PZ4&STf<649oXra}3>5Q?omL(>2z%3 z72f@_1ptxy)RCe1UQ}tc#qSXAaBd2{;EHe_0IhK&fSj*j!TtDkOh!-c0d><`T|+AX zP;?oJ-3H6|xa-CHDhA4iboaYr7McJkX_{=;A~u!CHkB56{}I1>PLX%QJ_%k6a;NVw z@bp@dquPOdQ7(IQPVPbQK*O@|lhkvb#-aP;sLT0jd86!ebCf6}!;r2C_hk1|YUt&NPYZ&3 RQ*{1wQC`jG0&mm7{{mdj0G9v& literal 0 HcmV?d00001 diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack new file mode 100644 index 0000000000000000000000000000000000000000..a69b28ac68bd547d49ad7502b1b3fdd42d74ec71 GIT binary patch literal 49113 zcmV(^K-IrcK|@Ob00062002Xo4tSiM&CLx0ArL^}J5_KuF~jh;B*xf-HUPumk}Qis zJlMTkgV&dQd2=w!;;H1s42lU+BavHz@?cHyOVZwJGR7}TyyD<}N~cl=9hfapawdCp zS{ns65Fc}d^V_uBgH*<#(!)Z0}m^v=PtMR@`Q z5JzowPX5;mTI2rYEKA*3`~cm7Mv$8ec$}TgO$q`r3_#(1Pm#SK>9mtpkik3X1yUzE zf&M_P3vcg$H*ovld!vg$ijL>F5_uBh;4Q9ceng9i(Nl7k!W59v@Ox~n!&MT$hH{15 zrjp^DKD|{f?eZ+F`FL+-0XqdEJXuF@zzNapwdfe~uQytCM;U%2^99@ZIe)K_0eGBE z)q6ZtX&(S^ZNkVVx0dwEODOADY9`rg$Zd6T`5>(l&YU?jbC{Vk@0l~o@)njQtX;j? zP|0n3UDmSemT6_zL=tVvEBB8Gnc8f%Xi&Z9FlW~P-_P^>ZqM&=ba2>Br<;Qq#6zNC zd!E{{tjS~31uf zA1Mt4LOdA2F_?57#dHY^&~_Z)A3~Py{)~= zrH)kw6#|feEhgZJ?qv0xTSy1(TyqROP#D1M;}}s;ED-=4WI03Zj`vx={-uhr50-$o zs)u*aUS1oiWNa8hF%T!!){@Fg&$M3cNliGMYfMk6NOnqpt5SR%B#3M0Y~ge`tIxGS z!^TgvZNc3S^14$Qk z0}~E#oPHO2$_-SCtP=3@kV&Y@7u(30zD6;4-cOT`%%OZEfs3oIl z3}Nn+mVRt?*)qw+wm>Pk2o4Zc7lAk+!4W=zfAvi-Pk9B^T915m6onX@$AW&$rZ5JI z$l;XNjt@w4^mpv{RSRX_+^#4)<#A6%U=bl61PoifF|zKz&e`j;FFUz6?^}b5O2Grg z{ZSzCGc!qvX^%QK$hP{y+Zcoy{X9rb>Yb zaTpV!{FT<8ay`QxDm!yM0B_5;$C1Z9J?r->As-Rr0LqGlA)K5j-2={MooaH~HhJ_& zbkON7e)nj;l;i^f7~_)X1*W_bBy(V_eQRUWaBuwTQ~An1&m4+M#;=E`? z$c9f7R2J}s1P&549gwGqiQCU4)^^4h}KJ_)Syz==z6(xreS$?wZw>$K-ANtJb7l#}&il~vT zvu{}QH{$_hKM2-gcca{=vw2sDQJC9#$JgOUTjxh97KIZ^NY>-ZT;SYn)s%UP#^;Y{ zdxx8uoSRjY!cbz;#N?Hr*%2SNzV1zjhG`1x$`f{@se@rk^)Cs!JRq$vuD#3z7Me6o z#gqPV^p=LY@vBOL!NhU{S<8#WRbsrc-Y>oRvNlA=3TA8`FuT83F%H@KQy`zLTU??0 zH$HfMfP&T5^mbV5-G`ER;BQC@6ODbnK& znYB>x&K6_$5OwbTKDNq4BVwZRgmU~d!!`KldDGygSoaJ5gHm6K?lh$&Lh|hjKq0v< zMs7Obbp>1N3iH7~&(86!bcXyAl~4=|+0?f9=H(E>S?x<1JHvM(MwbEY&yGMS#seJz zK8i+3$kE+-u+(%v=g`u*JDGwjy>p2#8(M@r!sOG56}h36Jo{gW2A1zj~W2KQ6= zVN8^IbvQ^~^@UEg=7(O=jDwFh?ikWkH%?tAXQ=?WR_R<(*rg4I1(xb!5!1sz@yy$zl zZabS7BeuQ%X-UOkm>LH+J#DY>yo{{wU)cK>rpg*+_D&|N7$JtOTAgQZXjdC|lynt! z@7f+rpRQ91iE#Pdl8}t-!UWg)xZ+x#!*J-aBVt$=&9m{rj`n zw6^R|>|C$i>Yh~!G9H+$M4Dl~o|f;6TmGge>`q!tSfkznQ`VfGgOuuOK^I*^SpP;7<8l4T>*8AK@jT28Qowra0&H{xa2 zw#sqai1#PR7#v&^guG6*$cdC*Wtz3micVY#WIQwq&byfEb@VTK#c^RnttU9l7h)}^eh+r zmvU;+MslWA_=SAeRYPJT$ZTm4S%Vt%iAE1%1m8sJW6q7REX!bct~!c*myMBsdCoD2 z7LxesP`2?&@`P}7+~yoNLxhhTb@hgZ`(-}$G&R?`Cr?CrGyC2W`?_FIKoxv%>&~=- z2ALRCtt4!P{E2#J-w|jX>l}8<4J{0zw+HSj>Q(js9TkQ&o`Vn_@9X@4CItSN@>)Gp z$=xY;qBiL^5nY^gAG?C&bhh0NSAhD^Lbv^a(s6&N=)#~if zl~tXsi}TIr75@iasJt$)GkBbRQ^9V+FbqA@udu`cT2Ug*<=sFkj_%!DGVkN%oPadjLbWSAT0=J zwNwrbI1R3*jBdqShhUxQ3fCEgXc8O)*h!ah^^{u$5W&4XAT?yLRJ#}qQiY;E$k;QH z&R>w~T`e4!71|iItyZOFTIueB8!Ql=PUAu=>!7FwC1W7Hp_c%UtXzo(I_q`f2gh&$ zKefm*YK~9O(|<9QXsicu^QAV>c6lQU7FpC-yq=-QkZ#d6&NB7YtdmW;7#~6y^Jg~e zsR*yZ%lT^i2ioiD!PE72kzB&p>1q|89ND5OK@*n5@3q0cV!8=ueBehL#voM5r6`IT zMhDS!ID$Ryb6yH;8Vm2_8@Xh`|D?5HzaUi>o#&#hIXaIe2^>M8R zmqvTCY4AmEkmEhsm%vuI+HXY+w+SO&mmy6?R={t-KL^(L4JwolqPG)xoMn*94uU`o zMc4K#8rUcy8rSYzxiIk)2+VY(iXAc!<0JmNfEc5$+S}fqHci=xCWGtkZe463?Q#}+ zOGsH%6E$ICs{^SpuM+^e1agAPiVvvj(qUFLwkpE4r4@h=E~Qq@U*ssB_d!w%&wx#nlc#A3;LvH%T;y|Yy&z5y}+0f5Fw8qLF zJ>Tdb_D|_ODT!u-i){P~W(#-%_-SZ)xiWa1ZIZ!m+b|4<&-GIf$i)d%_W^e5+O)ZJ zhrEEH$doN4rT|J#nzx@QIdih@V$i>+kNhdFoTUye{M^6(dfTo) zci=*%l@l-%Gg3TVA%V;hJ7FZY0@5Sik{48#6@;icwht*1dn>UZ@Y{vl{Cq7AI{u9c zgP%@aVS!~oDsgSfebR?hvfJ` zvNlIBdJ6cD$>`yb+c2K2>nFxev=O<-oXa@+h!5r-N1_WqwJMl7Pui%RP35oarjkcH zd)Lh!w~8VLa=rYp;_oKU@Gcb$NgZvySgexd z;;MQ5`o*-Pep)S8o5kH?y;-h)V~lzJRnOntEf(wR+eHw3*)!JfK74NVeLwjA_9MT& zzWY5eewec7DJ!&L(&|z;$XO;V*cr=85Q>SyfnkjWmOk%Py!r5HHNU?55>))y>wf!g z`F_rWVbA-i=Uva|i+P0e^E5%LAB6$sEEA;xksnwsj79rG;r~2`{MVt)^_Sb*#pZ7L zmcL)zY=W9!rjvpD995u=p+CA{cCQ~*)MSUsN8(%Es9cx>@2q$|Z-kW@BdDel zPHN>9WLhDhY%dH;YlA`gil82}%<0-F;JI)DNsY2{r!bCEIR?U%2d*1eH!f=581Vqc zijozj*adJynIs2g;nmgUBuNslIi8i~6__2^{{AS+`*z@33k z%F3!PWpUtpjITC_NJf&5N2hDewaLM>1hL~GZmkkhctqNkdV55Id($Gk4hqMO-0fZG^P@onDC+B=3n3yn zqG2gw3fjPCdoWl8wanUJ7Sh;OhHF(~lw$VgvM8QR9P(NPOP1-ICiW1_Dd9$ERAC(1{0u-pS_wMU2fjKZVm|E= zk1%7Os>}y?#Ij8ogWVhpzG(E3a@7)iv5*=o|AMwyWJw!odx+DOLc=2R;h^0asJX6P zlKNBO#kwZ4b&2ygjv)M2)ccVzrXUjiPn5GYWD*CbA0)O*w!N1%7G$Gb0M07$wYzso zFE#EO;vmpdx>SMg?NE<~cA&*v3x^ZJ`n?$MdmAv&`WnC{Ir14-hV>Ub&(3pcu*c2S zt;d2wO(Q*)TBL+>`+GJSK3+IIJGg&$Bri@GtHtLexjLoKmp3=~mRW6*my-#4iY*{5 z0{Si^Kir`PdN28{c9Ht@4>uSu6nE2$<9(i9vU96@I-5MulSY=n&Z{~lqmx+DU{t^$ z6>4iS+Ec@LvX|_1Go$IK#@7e9rXHvvnlQU;ESVzdq4VV5Y+#7S2KHDQC3F_>b+Nk` z3gG=I#U$?6i9clm_stflTR@2lrhPn!jn2_d>J3q`oR3NmaFiPCC+H*kr06IKl`!}1 z{^V4($O22Xe{rNc{Pw1MasOaAD@z=f(%)(SR3zUy^%=btPiSdV$3}+|DvqI=jn@Z} zL&pq(&=d{Z<6`s}WGVjA%3|T)xtC&EpMEd^M1Kb#46z)gu|pD{u1AQD%kB)&|D~Na z0(ZK#a;n}7`OOzvR%HI6`)$&0z_2|pyF0aNf5QipHp2*?(bc^)1NlE(JgK3{Gs0d> z{sFYLl92F@0Wt85KvuH%8&3E!lmCbI{Fc5%m(57^ug>pea)Lz=E z<;JNuBoLJERDxsuK^bge;~qA47! zLAA%>>g0k52F^Ntt<6%iTNFJ`Or7F+L%y}02G(+M4FxmvpCG(DmfBBZc*rgAf+@@! zTS{Lu-KpcX!g*JoMF}T=!CC(4^6F!LT1o{cA%ML}16pRhDDGZnZ|?qX=TG8UYvZ|x z0d7aww(4deJCRsX%LSF1E8dJGi^Os*YKl5tC?F;U_bj+aVGZKibbf&QDa5iAuo|vr z;}zrH+X26ru!WM9#x~p&U+~h}F|MZ*VyzU{BsQUph7AH-Q+3!1(o=O2F>hh~`PrIx z+V`csJILOSVHekRdxvyqZSYz_GS->C=Dvyx-;!m2galdwwgQ^jT@ ziQPyQM9jP~YT$d3H#j3RAM+D#3NvC zxgF9h3-Whmp(P;9AqlPazNHbz=9U_j74v~T4osPf{EFugM}p(8%8^11EKUDz&nz^? z)245@RAJ)~(Ewemrp}7?*D0QJNA<3;=`3lLHBD#i`GhTq4j1P%%rVo=f_SEJv{7O@q+=Ib+VhI|6DBFmavbMQKTzeUw`&W0CmvYE{Y7GKe~p#l_g;=! za)k~MjA@nSP3q@PkHb4pvb3zj^5D9fdMT<{T8XaQ+kU;~hf1-6Pr z5z19~RgbX1vdsPg{U{?~vnP0*jgdW1!!Qtr=lEBgl%+%&fHfNei2<=OAr!gxxv>;3 z2PZ;uptn-vF%88kx}^xgaY?T1LJInj{*M2U9LWMQXv-S6aO zB0FL~pti^vcIs%&$oQ3f@Dangi+qs=GP#(!*)ubWW%ms7x^PIR@fl$)k*s&>2WHpWBDzI*oQ+dWZ`&{oJ;%R-XfBD7q8~f@pjt72X{EAKYVdv4XFPvS@=7Mqlnfss*<824q8Hz1CX}2%6 zzb9!}lvXjUl(mW$LO=B-tc@@C!wVTAm8wd918Ul!U zG#cHF2K+-X4ShKkXYI3h9p~NF5AY=jW$}IMhamLknJ>lD>mc3rqQ%lpce6B@@6DcS zl}Q>;vZ+}%bCVgCltrU=?L+5Xr+wZTmW4n6J`W~)vo{Dg72Lny><^st&PA_1?l`A| ztNwXQeETN;r`J7oe(enVSC`^vad3BdbZ}6R48P`XDo>*<^qbyJ(h{QCgr9$bU;O*0 zmU#0O?c1>PR{PfLzt_6R8#fO0 zLTKM;9O@;2qtQ6hO8`%!@w|pwog&4H{cVwjH(|62#Ug?UYX*yDlqA7qE(Hk0BnaI! zh(a-Px)p;*DZxFI%9V>e82RX~NC;sCMl!Du&G zN-vlO9%I`R-IQ68MA)og2sR?>CXs&+gt@UyO7bT_O)QsCCc~6u02>F)gsXn{lUPMD zjHHQ_0ZM2crZ63=z=yd^qiAohwPj1m+wQp61?g0Csn?BEU2}n}3VwA#}HpYKKS^&8{|&e<}8 zwE$CwO}z-D&P(^iNG?;km;i3r4_@R98c8W;X}Wy9zmJ>@+|Wf-`^+3-_CE4&H_WhY zVn`zFJ?*RB*a3NRJ`9Fq8d5G)r)~l($9Gd#z`mDxX%?3YveZwo2)&skwAQ{l>b~u_ z$5+D+>|w{p$A5isbo{&lYW6AUowKLdfblX0XC_@-F#p_YWTBc^OG;?~NYmK~*6{?t9xwlne z2KHHDMIt?Wlp&Rs!}I0seh^0cS>W$q!~Z1sT?)`h+CBgx>fJzdr6#p8(;jv00+G9@ zZj*vHu4tlqL^+5+2L!kXK_)@ITVf`&I7r}gp-K=d&;WZVtXbrjH*>>l>E|N*kw;B% z(?H@{0F0C1dKb51HxyCKniGgd>as*_C#-jUc?Zp4S0)wLzZmhN%k{3o%m@%BHxBa@ zrQsf#TIIBdKTqmv7OKYJI&j%8f&tq_xDVT9FfVq|b+rg*O)mom#<(vXemOe$s#kyx z2h|7`;WY?(gTO-Av%XyPI-i{OaM=F*^+dpuTSOoNz!nq~V~hn)jQ&08cy;53@>+gC zc@0aTw1z8)qlPh1T*DhEtzi$8+8k2FIVaX~u4eGtp}b2U&Amd!b)0_tvUAxUy$1{Y z;O<~LowoGa(YXDVtDzRnK%=&_5ux^#T}y38@46Rbad@2k@&8 z=iosoWQPYfwqfUT@M~UuRH?rDFdlGws6MV#x6jWz2BlD~*S~9z-Z4PlxrM%;L*%_5 z6n4v5w7h3S3ucHcy96vO`au?p%M&GdqNXv}2N7UR$$V4R@}4-&Qq-g&ns7?j5K^4W z9ijFsbki8|lP!yVz-{oqc&f?)ebb3=ZDCW;aX~o8R<{w@HXcx~tCnV+aQR8PBkC%W z*lTT7G(3&`7Xhty-{kTP_Mbr|NFi>c=Mm{vWFZX>Mhhu0`a*1NH5yoI6vE1mCD;xjxA-QSghu@Qv$@i6 z=uezwP;=!LqHc<7P32ZS`ee=!=F zUuht@fGcI$#JyUhH!my)CR^pY&G${MZFMW%i~`#tqp5MTX=ubB1(ueqq3DD;8#fU_>i&zI`*@#+s(8t`sLM z9CqHqa;O_B3ecv}@Gu-y$T-Pqt>*LOj;J3f=oxVIsSMgBsUz>Cm!6zTvW@?=7?6DW3Qv5(n|{sC4w)z!AiTE z`hJR1Qx*f24KyMsEJ4;$H$P{!-}cYwTu|lQ?x-{m%wm{Y+ffhEB-dbQ5eTpgS#VUq}%qdnbm<6H6IPt$*jixCpf1}E)3=H#P9R~tqduV{=jC6NJ6LhIVhQ#|`5Efb*MPkge*qDT zB@~h6wN3c+PH`v>+9}|21Ze=?_U};oS{#3=K1`32p`VNdqU zafibgLlAQ0H;D=-9KcqD1!To%7;yEYn`~JG-}+I?-o%O`u`3)Jy5&(G(4+7GP@RCy z6fBl=G{mFqde+(zlT7vU%;(TT-ZYkEEpi~6K29?4MyBNBdoBlZ6QKNtW2_vAEx?3a zEYo{Xu+d@>g{?g?ih%fM&>#bXCXxa#X;coMVghOI^jBJ+45tSQq#+5cow z;c*F_Q1l1R_#K{yoP(cuHn{xI9(PZ>z3%uk7`$i0!KhfH7$kx2*^6lw(zHhfEgLM1 zD=<)20VCnBLQ-)B*jRGaMyg^>Gg^|BHdv}DH`uLwhHtj`3jsHOEZVhdV99uBF5Psd zH{#N=<&I#G62mGEQbk@fl&h()w91Z9!M>IPv55dw1h3%&h3L6)Nqqi&J^fj@kTLM9 zXi^lcZ=>cBC2$!fP4-Ha$=p&JqNvsACCC>4LQrjveBbap(b9^RcrwwVI>yO$1_MCQ z;im%CXN4AtzLS7W2jsN!Dh(r6nG*zX6uws37ZUKRJiiO8axcKBl=gp9{;rbm(-iF zeeFpeHcuU34UFoVN#=3q@Fw2K52k_i>#^pQ--EN1aCwLOg(>7a0`_QB{f4wFNOOUb z#EU}Y;bY#kJhYZ!wh+d}^BSMAbmt6%v>4!Pu$L>&p#o4WL6DpS zM&tlbg#fA3>Hpdt4*Hj!{#ZO!g-;ErkAvZR>aPl0RFy6CQ>P04z3U*=Ku@a}f!59$ zwS-@T;pY;XH{?nKF23mYG#cIx+ZW?1C>NWcIUDpZx^Jt{Twa0e z4bIxVbD9P1qLDF=-4u39u;OH*^Y3C%0S>}x zv|k5ZK(f)}d;pp6eiBc=An@0xg;`CfBZ)I{W9h^ac8dUy4IOWKeX<#(i56VT1`1#u z_;@UkI(`saFcibz#=2D$-;kwJ*O9_ez}Q&Gl~%Z9y16f?SIe|G5?u=jdqMIDFl7qC z2DBFBI(DaN9SENjmNe^Qp?z!PC?CTSvep2YHppu_bKB4hS>Z_qz21QAN$qH*-5Ym? zeOgmB%fGC(6hXH$DuqtRPoqPsFdf7s6(%HrjmLn_A**A^mXBf(L#hejsm}R?v$Or^ zRvY?Y%4t-!Bf^eo>wVn(D|9%excB~9l*Fi883)t*&A9Y3+VpEtU?k^z(P+vHx0h4r zzpuKTu{gNHWDip(g~;oPDT?5JVj>aFffc=EV@>OT{4Ma$ns*c$zH9yE@MUZJ`TkLH zyfu{Us-Yh}$)=bVLYWT^<<^Y@3P?P=dzL%z=&PfHKa1v^z=d3Qfx>5D{495J zPWXU}QabRUOce_+i&gd!^$a5+9N=(2qk?BU%4^-GfaHcH6sudE~kouNLBT-K+qNgG??$dP}vEWQKZqVwQwlFJE*yf zdFpUl6>Y!(46jAg<~9_z6!ahX^AE9_p(f^oP0=kPP;?rXC47p;K>*xN;{~S@mJuWr zFwJ$YiyoOO-gcSmT5MXWh(q_2jp(t3h4M_ubvBxt-txmFTdwl@9?fP-7gRN!F`l}x zPSx4i6qU1ei5hp|!l1@bzKCf~m=63VMsDl^uokQ1{M_LbssY%}13ZEkhYj>Z2An7A z`^I9jz=s{MNY6UrsB8g7)!g+Y<<0Dusav&3M(?{HobwC#c=lc#mpGD}U@6Woa1_&d zfXv1er=%Xq->*inRe$)<>7R>NM0$Ivl1JzFpuJ!R-Sb|*CNZY%v}LUZJC0G$(RJre zls?B{j*TK>cBE{jFw@64 zD@QeT-M9V05S+N-a4;M-j_vsQJN4w^0kV8Etti{u+dJvxmWEV~Q)j zbdKE28`{RSly&{m;dU++Bt_<^@cbK<0vRuLRJrxyJ8AJ*18jrDH@b`v2MO#K@>Yg; zSXSf@VXAJNNh_b_vOPX~m(L{uHX302FiuR!<=jM04#v#kG{Ja2DhbRc1ND$%nvsZI zFPPu&dSHHpg@sYq)v$f`+3AkZ5+8qV92^~16%*$ZmfAl zA>dJMJav80uPC>Vso$&>JrZ!=A`e4$<^Iji=T0?vgg+iK0l4>NpvRZ$`ZsK3!fO*y zix>CoM=q%cM`*bSpfKh%scK=PUA0zqTgY`Qe%QGfc19q>9k5A1!$KiP0XHn5I*O&k zCu^G6=Pj~L6e0MJ}d*+i$t^vWt``2f~SZlmHF4)Yy3iSvdi|T zuXNI@4GY~Xr|820quj}slqgi~f6W%qdI%xiFY*5@LOcQm54C7kgU3=$1qsr!%8jl! zb+n;jO-RR`(}kVfiEu!Gmsx69=~hqkD3K41Nbe<%=JSaQwU2AVx!H4>Vi<915VB#1 zR~v38yba=ru1s)tKINODwHsV#F?(zai3g@Zval9OUHXtg)o)pps*PisfeC4fj*;%9 zt5H@#=yaGCQ<7w0T4Z67N}*L^a74yALW(&W_4DW&&x)}^iM(0Aa^ui(7ZVtDlqJ>n zbU(`1; z*d{0HBr9>JlHgJIjoKF-=WN)!a4y=tQGR}6FRPlxO(i%qU4R9-x3||4j#HdnIZm_5Ocu>$40ABuft4hV zFql_8K>APP#6hrWpoO#RccW@M)JuQ5T^XTzRBL_6&=hu=;I~XV?qm|n+w~YM!=er@ z%+O{pF5^`5S%btWU^dXI9eh8*D-MfglPM?~!wej($(hA0Fl_Ol!(XjB(t9>K#>zE) zk{v^xF7IocA%`w!tn1OqZ;7az97O|F_h6s%*zOaHbCjUxn)rPOG;WS*i|?eR&f+G) z{1jP7pqHhL_G2UIEwMB}%#k9^a}f$NWr0FbVK#Ct{1Gn*g1B)WI8Y(LL^T%uqFldQHg_m9=8SD{L=c9^{o;lcMsN_Q!mV*}A0+XhYZCy0JOM#Do?%bbZXe zbd()fX{(|PWGb@NLFw#s>H471i_=Y*r{?!IaHyZU-uR2e7X)ZeHKdlqbuiDUpK7&^ zJ4vE>rY!U-er?&~_b&%u@h1MW{b#sfbZCk+K7j0`+5-IdSi%az%*#hNTHgW)FlTSV zrxv{bx;`IEg>+%v<=vG$DPTq<_?qDk$6nMoI&{zgP3MPp8fDs27^YU6X;FeXULv+# zy=^|zDz?c@3c3>OQb8%J2h>(A+tZp=y5R07+0;f*IqkN3soOrwuvOX3zu5sxo>T1y z)DfsMztWS{O_g7jE=;_j+`K&O2A((g?3*S-CF#?}U^`YyHZGA@m~WdWR+7^ zZ*+=+dl4QaX!ENYES6ZdV;MMXZ`y$rbjWl2>0wJj1-aN;uL#^@M7pMbGTk0*Ro6&A zU)|i7ie76FGkFK9p07!KIePK3Vy2mIED*G1VR*W_H1K&}$-vmhi@zcGMEF$8K1Epo zY@gJN`Ou0U%e?INyO&p&PW$w9*!fkIo=l!js49LkIEkr0nl-7okj9T##d%W)vm5wE z(PNNkh1=>%RUOhh zNDlMP;3zsK)#B|K1McfMI&~1)(v1_~D*}G`x)yRO@rTwh}{ z*)^Kt0j=JuT$p3RV696zN7bZVM_rdrLp^{?Vqu7?rb;W19g>jIy;ay<@Vd6s9}F+s zJ>@}B$*|LFkGsEi^6#T}gW)*;l1CBgOLsJQ_444bXn1jU^y=t{LWhQ$m_;jxPK8W9 zp{>Is!cV_Pp;VNYR4Uz1b{Z{VzJ;zK_c`x~Ep)VbXn|x?Wwz4PQ+OV)J~r3FnUuN9 zWgOiFmCjIDO#(}*^3bE9v8c#Ms66lV#}sUK27O>YJvSh<91{kh$O6Cz1%c-c z<+2{vsWkLW-7K9&F?$p>p`{X#&#%z)+YCEhfPdnHZBZkfdWl@P!Q3n&F6Dd1dhC@% z)iubqqfauC@#GgTAX+XPy1we%Gym%FqY%ZMld1li7CWCKtIuHs)X0_kW~4d2J~VE< z7Ir!i@jwQYne31gzMUt~v6&rJvkbKJt-`A(r?uFC>k@fQB~zX?=Gpcx7?pURv^+9i z4`aOXLp|hQuzLN2KX}tPbs~SFPM3A942CLdc}GE5$=b!8U{Ht!T}jm~DKxnz6J*8n zDU~)ly)++>rK9PP#YFF>9Q^asVpnK)w6~WIjK_SY)26tG2irb6k62LLh>_+^WaAEMVfup`95OVzr%lf_>msCV;L)=_nQQ*WS}Jo#S>lD zv_AC4l|CcO6aDEgkU4GU@LRn+VLCzGt79C@mn*$e?U!U^+R~&cDawrDk6{!}LH?j} z<;Fgj(cwMJ8tSM9?N|8X241Ga_k^xS`mH{_c<|mT7CFvI7Lw0F6%5??4D<#mT<-MIa>sKWaUrUD>h>1+sH zkIB7}<4BdLEjM4!I2-i)$o)K3mH(BLTIrm#0Avv?8&4)lzUzk}th**>1${aCYWA1K z))o$puh*ua0!abxj|yjfuUKmZ;Os;|JL*g?V6dx4VAWs%2AJ|JyGq;-J$5VZr*m1| zDIyqhipwPI<+}2 zW$dpdp1LiT0!EL33xHzGdeNY2>GW^YlDYng1OiyG0B!WDYTt}BwR<3f8% zn?N(aD~~5_K7-y4VaX~S%1Wg8U@dcUBcnvgdoT$>xj8%F|E)Xpc$g#Gd=g{Y-X>3e|^d%unr_mp5%wa?NSQcO1Pe*AYWOP_*gjwgcr8cDXse|9Ojv6%lnClNU!P2?HJn7z+@V;>CJJh!?G~0l(>orm zTXa`?ejFnCI0jamISUZd;MR>d4!^>TwJ6E%#`Wc)H}WW=?5LDXJzVHYY^z)Pn*6`q zZ+%2MMlSr`q&f~6-qy|MrZFe?nD0|VsRnaBk0$QCZZupk;~2ycJiAKYMoMG*6F&Yy z(ZS{+JqWCyap`)8!ew8y2HFvb4NI<3EhwqX1T`gw{~UR@oCjV2dmqfuErvxlhZcdG zUW8GrwUdoG)tvpV!O~seftbnpQsVP5p~q=V*28fN_kMz}j%DD00Q3AnHtFS!jp9#l z4^MqI_|bPDE7889%TRX6$+vwoN+ipVH8cZE#a+~0V@g~vd=>4eK7S-P%5>?_u?bt8S$XBG4Y$Q}Jr zBo2r2fG~9V>III64(_Cf56r%L(Oe_4n2Vqe%-|DWpC%5_^3z(<4Afe<7+h3mDfcOz z!fvT6Tway2{NkiD0sZITpEjs-F%=9wI`aUG(kl;Pa<3l(QHJh1C=9<$WPJceBY2wD$w{`8dvBEkEH&7ul9w&n^Is2FghN&ZoIK;cIIQ3%XCW8JF zjcEm|h8SYE-9Zk)RFs_dtoW8#`>(hzh=1)%37>VJg73b=*UC1SQ-m+Y;6X>_xtoVc zt05qlPQLliNwzA*<^b80WF7{oc7SQS_#FTP%AlcF^eJg?3OlLv^S=v)m%eeE8JU;9 ziPh@QCXu(p;h?R2Vdi(Wlk(*}Gsw2#K^-&tM)#3}>C})yTKiDw&@CF{>LIrv*J>Ae zns7r_d$VAg>YT2);cJR31mBGM#EXGE(|fA@+0?n{}k^l^mu zV7fgF2ai6q&pHk5-Ud`snRD8?K<1Czrw#4q22|stWq91$9lygUHbWfi5NAWDJ$5>S zo+i2a9gUCpDjlar4X?nSXjUltRzpT%iWRIWxj^|P#(P@pfY#grEWp{>`p!mdF$1^3 zDx?cqq1i;+6O5#uRYK@_#j|MzD@(cyYZc0rLCoVhEH;W(dcqEul;dCnSb0Pm1(x}M zJKvpy=2>&Cb5f3=blh+W%#&SOf%%fmmsd?7dRxD9%TWo-oG4ShD9(EmEO{>t!u&k| zzO+c=GM9Sd)MB)jyLwrQPp$Gz8vk<7+`Z?mG74)H32&m6({sj#jNtF?Ja7|--!(Qf zv9N-C4tg~b4@uM?dqlSVA7&k5Lc1w=oSo89Yr-%P2k`fPibG#qk?O32L7&EOY~Uu0 z4TK4yH5U!kM3SrMx8J3$8y!B#p4z+S|GSi=ua%Lg;Bhfuyo=|k&xJPlQ%9p1daH4o ze%)s2eRh{kj>l};zU)@x9%v7f6p9m}hj{b{czsyTXKyf>1}K6UKDNC6kUO0hC4wjG zI7VCKURfbn1#4ScmsC0m8XgIq0c{92nndMM)ZYb9$*YpG=nQ{)WQTEWJYg;a6(a3I zjv1vlpcR~K1qM-G9-Gv2&jUNdT4TTuPq5zR4r2b5A#@uI4&h+6gm_)t@D6{m5O2)F z+VUzWggr@HkpN6>! z5Tp(oAZ`jbK{o`2Kuaf?i$rQ9mDC&d-*-n!vLwG`8|EO0b>uy~d+vp&9EnsWkg|)L z`TXiO`ZzkmCm~@Y7##^2=T!pirCR<5u{~bB?8vMJ=u3>%%PQ^55{Q-N*6keOhAOr! zRz)FflnZ0opQ9H&DGP*!Vmgrey@7=9b~^fNIzZ}xcth(yi1MgJ!h*_*93LuaL?$6& zLR$PyAVU|r@?6P`C1~xQ^GmKlTC-Ut4p?A4OuXWn1;{`O+FOa{f|l+$!yh54!I~}g za;zkRd+mCMU6+VBItkdr%juq(!y#7H5X&=d>aAJ zf?4smlj%XlNCR>RGKT*FBFZqV`FcN6IC?=J<~|h8-zxzY@kjK4X<-Jz9q&l$koj!% zJ3`h^J0t#pU4XlgCPP;1{OmqOLK@IJ)kC&RIdEF#khBKc!$r$=LC4BP#cfw`d3Sw1e4Z`3lABMn>#I+*u4pztzv^k-eVHvTZx-`& z#O{heefd1QxVpTWeF%pgMk*O`6ALlMr%eHKI-o}<%kEf-KFb}i~UN5>&E8bxB8QLP6P8e8} z<6to~ge8qRgU#5HRU%~8G06&>LqsS$VziAA6ef$xdf}hg*V4ST(&uu&vGvj+UNsb) zrVhQ&hkY*W@t~PvT(wjC3)lh*^@vq$SHM!COBjOymZhHzMn&JIa_mbUbp9JLgXVBC;kOu0fugjdrz^p1&t}+qx-0?=_coM6! z%m8{og}((PL=S0GI3BqcT)I6POqE;A3%^^KpZzBMa9rdV)wN z@4oO!14)4qG+%}6X$2C|u&17$vNvHf*x|TGp~Bcfj`Jwef*xq_0ClJlhcbe~ZRh~d zWcEqtf)z6AxEi@VyP0!!Z9+Mn@lH&q+N!nqDbDP@=D87XjaM~Ne=_w)9vW;1qkJz$bWInaGl-TP@AP3U z;3e}m9A364eCLUvVv|3UP}lD6E;|`v@#Az=xp!NVygsWh6ZYyA`(Yq^eOABLk48rz z6Kn%C$k+dbpo;z*9Ce_YxLJ6dZB$K9+%OQm@2?o`5d~;3RVCB|f1?8LId?4PEMECW9^Nz zF`dK?j!ri*@t_k(Fes1r9lCz=v;q0)PBG-*?fVaRXNORR+{9r>1i4j)@M*&NEK1eQOPthil?xb;&NWc+{Wf@U|Yo5qv zOWhx+iOoFmTI)cwWEk5t8UM4RaYfgdM#KOUU99E1k}mNdpDLc`?cjieQRqnB1fnc#+HMsPMrE8C;CN@!iFM{dhPzLE`)M#{XN7{voSbkuwC zA#}n^Y>VD4<|)_wT~7W^JDeKIPM49giz|7qi!8QM5cgije^|2n!MS4OEb*eEfzlL= zHg#`^IuSXN4|;aqfi0Fj{pw8?j?q(j$byvEt(_TTByGzbFz<26XRK8$Vq6iAip_CP zoaR!qWfsE?rG}BGeC4RT+yKfPh$7Brl>_L3Iji{P^ZDhHAA8icvBHoqi*;M)xgMfydfoU=v86$}-uFU9&FhhK@gc78|H{n&&*-GugE1>`<=U9< z3famI**1Gwj^O4(5N#V|$fEr9&2?sy8}cve3giO1EO?xSQNc>XKoC7=zha=58Y(5} zMaijzN`|xBKYXri^+0sex6UL??%pqIOqe}u&3X}yXT}ue3ztg_TqYnCFfi?)LNLbJq zuS-M2En=1O5(yA;b-#!Wxets@y?n@^_i+JPON&6Ri(b zvzTP4oW+O69cSGrCEe>)bREAX?0a`myRfLOm!Rsr^zA9VG}6^Wwg{p@MA^%I8pMpr zElOu#czoEsit#Ri2@Ds%Nd94RtekDiYSZ9ZkqMkgX|LSU{*D_T`Anc6&LKe5W1UZ@ z@ubd&)x~*w*X{m;uVtytvBG|~|9@PX;^Z_NQAaEG1J#f0k-isroSl!)3c@f9#?R|h z1a{HVH|R-p;7t(kqiZu4|D@7(B8up3Q{jR_#xoZr_i zN7^$wAWjhJjtV&FiPtcPY)ou=54O!Lq67<}^n*k1l& zcO;P`JF3MC#$kkax!D1DoV8X~&IK#OgFv^{M0(u+Y$ zl*K|KRg!9)zJB}8kd!RSO6o1Jxj1~xH?Qv->O_>X;3d3%b9?t{{Wd+pg^+w$r_VD` zE&JSYP@7LYw@@k#1+OgwftEYSo~Op@4o}?f+U;JFVG0@KnlsDSqoWJRRnvBs2m68{ zRH;=1Tk(ZU*mPydHQJK-T_p@?ZaXavFld#KmTRyI7l&TKR?LE_R96?U;m|T;c!9SG z%5)pUF%M~@pcYHWSzVhID2H2W9)-JDJ_C3K@Bg{I?cEdF5EkMq2_+MCr#W7|t9X^m8F0PyugJ9RwEEE2x`Nzq)04R4fwz-qHC>Y^O$;eSaY zF8pC+$K*41dl6QsA|Lulf8Lr@ZYNY8YqIQQZUr*GVGop<=P4#yThZ3obkT4F_rg|C z2r?$!c^KnaN>Jp%u!cv0hoT|pZ%TyUg+wW9QiN3UkFQ+g1ds&*5RT@C9Aj-lCk#m$ z!>XCy(!8jcoI;_g^s*jlnW_#SP$FHPbAv;HIe%mI6Eejeb{Na7$Whrf zJ?()c`wm2d-spTi)Bmu&=fK#;>g}9O*xuXxE^E?nY4Rw6;d*YO4rz=6NA@M6goPiB zfS`$w-*+a)@wZkmt>5*Gsv0`IFryQ+kq4A=6CX~;5)x83V${}N3uTGhHSFnj2dzV^Q@iYY$Mn_|2- zd_tkcIZXuoFcIPZLb<7c7CqeA8Hv?h|9%Y%Iyr@(=ew8ZOSrlML8k)FuHjD-F@E3) zY;w3px8U^D_v8vF1-#^=M11ry%@81d%yj#xxtbn!lm!Cl?^4Ha$0Sajt^jTv_(e zm^ij^TdQ>*JDtv%b{wC_6JsJV=UU8YIy^uww2<+1I5h0967pUPxiKc)YXY2g^fIn3 zd$OPCTo3B??LFi|59v;a(}6MR|6&sAeWzZ3+hmg!k2!-Kjv@h}uCGeij^eP?hNzVdoXF?zP$df&>Izbfhk0cMTfEEKHM4nkau@pIC>@(u8d}yAQ%G zOIAQ^$86!dOST30Qatbgvx*ydh>BNX>O-)F2@Nrr-c9|4J-B|#SrQiIzo}O))pD%q zr;B=`nm_o2Jtr71$K~ zOas5{kHEuN;iur|I_c)!gfT7r2MzpcIv9X5J@f7eexZSn`vY?{aV*q+Xvq~T_&c3+ zknD7=o4z@At_{0q+I#qqI=D3$ShpY=oorlC<|hrjYYi-$BKb)7_fNu|Tg&c^bMU`u z;Mc|&yx16h&NSJ>&vd)`&uQN*8(zV6y9&Nnvicsb+f{=M16ym_BV$m2>vq+=>rb59 z{^VLkY7Va3RoncP-h^eFfL;DrC+EoC|O$V7@pd;Y2uX|BRHF{pA3D{hB};zc!zf^50J{5Yrvl&nfeX z{qIPQu7F|xsn-gJtMC&ZeJB#c$m0==@DhC^=J)PnnHq~==|hDO`hK`Ajk(Syg2g$NqHb(%>Zl(e@xWdEaL7xDy`l!IC|CzFLEt3CFj> zOEQ}ffmWfY^%|U<$x6fw{(xd^wj)QuWiznRaj}8t9m!gRY2Z1syrmhhAY{pvvgObH zFb0nxf}{kIgEd@ST1PV@B8FaIxdb{C&rCi8u~d(;!#fO?YeA--=TMu>-1$8~WPMkn zJ6P|1sQ&?)@*rJCeJB> zQytGzFIR}A-xEq?XVpAh&oF%x1~&_XuyPFa*q<1XIAq?4WFpC*GdPa@XP!n#T;aF# z4tQb3B#_k;R6z>HESh3^(u%H@{RfkhzFiH>JIAnXnGK6qDT;V>p;BYdT&NrX#}_I`s>#Fl3B3yQpOr3nCXno3Lo7AK_M``$vPY9=Cv~&WD1Q>*MOd;Ry(g6ANT*2fiCL#JiHW% zj5A29Vh%nSEX1;K3O=||F#*#pMRtWXpB3}^US2Bxys%0e47{h3d zt9G=YV3Wd4BSlsXUKmt}Ioi;kIWfBvbvaPpN;TNEZoa}U7RPS--7m(Si9K|gEgQ1U z&*mL6mQ902HgqoD^EzT7XpNk#ZXnyg`W`@XT89C#GsXV*@4+h^WUT#r z{_K_A4?)|JyJcGe85}#Mm}RXJcs@}_)4`jAl#NEp2D>d)VCSG|;jrHYVX#|MFVWuQ zP;KIF<5Z~0exUeMGtEL!$W;^KMP0JuheaG|(8d#buK}E8ge9T??}3_=g$pSy&jlyN zo{_kKz#}n#8Fy|yh<$oN`L#fYx8d-wNaFoW1+}1GYlnfq%haN0q{gU-Ox4guL!T8N z@yS|6Er(ceN=1E&8?$a(1Rj4>vm$|ovwuSaPLem=fIbFu)Ht8qGjX5BOJ)vNF)Vi zhZ?+W;`6AkR_aESYzFpdOL7|$6fCL23zBFD=S;!&cy&Do0|*aIh2a8%c4{sN>D)X% z8CL!1Iznm8+$7=a4fs*I!K-S-QLB?bTR$u~nrz?rv3@*-s*Gbam93uE2%*B-VNRd`QIzZ${BN01j&Rqjj zwyvpM_-VY-rEkjKZ<~i4r6LZz5JbdB9}VP9s$=weX0KK|F;29@blHvLUHy2^9Kcb9 zfSw*OwBSvO80r!71Vn(X2TZ=+nCj}fS38;LLDSKfk##!)DR)(Xp=p1nd8Q|1sf0cy zL7h&YUaIW2iNXzsG6g^B(C*K6`0XY;P22J!6Z3R{P8@IeTr752J`3Cz*9VtJN2HL7 zHD7O(epmg$#Izm5!ZkGYYfzX=+xlXTYA2T`y6cXns!|1blp}MYzI(_@8S_4U_~30;qq*LV(P&G~4KY zb@~UVVV(1QoJ46Lt4Ru{hOVEAbA18by;5)#>$SZYI!mg=&~>6aUvV$j1yHnxlA8vr zEc?>plfv1dli3j!Wm&5%zN`>kp70ooVJMBg$~so<_EAa+!YWIs-txErF4CxOgnB|O z7;L@#?+&=P{|9}nf1tSk0eGB^S8Z?GHW2=7|B7pib+X|&PP^s9j16!Sr}bL9g`4bs z5NL_EiAbbMQcbd=|9y9)EK81*R-Yu^bI-l-?noW?yU>SomECKx*;)wBd+=g3`VoH3 zrBHAVuUX2ZLeO#dnOMu1ui;{Ld3pYBISpTTpW(NVyt>pq?!(nq7}# zA{7bK!rntFq)77=f*(iU^{RGuY-h{q5hus z)v1nrO7?`O1>Zggv&C$FJ>U5@`wRK-`N0#$3Z$}yTg(@>NwIFQ?(CNKmx_+~jMaH$ zU73aiGZ~NIbGK8LRx-TI{bKa*xZ82x{fyZyetc3QhJMT}D?cJ>!(t&)(pXQ> zU*|HS?1!jHH5`hVOFOO@N)%SS5&v;qbTx`~W5-Z`Ft#OMbKJ>@djv({@ONvShUvr$~aL6^o=J!zY-g)CYnZ>ZyDlNezc3XFEEKi5K z_2b&e3omDHr{Qmtx3i0Y_P7T@;9rh@A=^E8@-vKj^*Hk+4%eFVG6LGzLV&{&NEYH9WTDmknyjo7Ci`IOkdjJsWa zM|B5{1>eWkP*M3ER4cbP-l@6*r>o&7qe7$`%RLhPj zQ(2xc`&HKlq;lY&hg9Qo9sF&3OMEZf?8ogK?EHp=n$-Yv749Zj<4G zHGa_5~zZH`xrB`n|qt|2hk6c;MWQygN<3MG$;{>UGSLwjd?^`c&B zb;D)o+&!piKWXAd8Kwl=gw6s#MW~Q#UHyJCTZY#QcdT3l8Fh7r9bM7ubsIGN`MAed zeP~oTTfTcUU68^*-A(|(lf#+fXfWSyjx*>~eku&tYe~GRY2vXu1$K($C*r)aZjcqa&=-bRX#f3g$rxy9oPDor@c# z@I@5qe?+q?`Iz$@uVsFvp#i*B-@%Fl)y=r#UFvb$P^>ZUSK^S~$8BTO+uOr~R!Tq9 zIxTJasIY1De02k7tZ~*jos}1Izmb+#J&2!ZOe0$X#RT)hU(-@P} z@%L`u9|4KlfW?nR&1K1XoNzbm0WFy}U8V7?tZBLx$`s|O)+ra$v+GwdqJOK(LU+Ft zF<<4IZ#sW=hh2HVHj%hy@VMq(+#O|y%D(=p76$DFPS&5%e^wc6Q!H$-8sL=)uhpNw z!0i^71H3D>XgJ4F<6H3F*G4dxhc(JunCMg2hEb`osgwai$O_4&u}JL7c7blnwS<*sp_5Y z|K0Dddm@u5WNbW{@v~qLNg=^;fw6j(Se6RSL0Y|L|AO918nKX&G9fK1Kn{XllE<3K zxP-gYX>`Mfv)SMg=9Pt^6Cq2zdXu`baq>t>?R*cH z8lIXlgqbMdVU@Z1+8l^$?8J)~hUV+YO@I2E))0>q$4RD~-W$4+dw?*xdge~!dXa)D_>p?wG1~{-x ztCC?YpPH7h-EbL!$h2NgC)4Pym&!Op5IhhDx9kqR<*W+%;yqQFy;*(=a~!+}bB}HC zP5?)^-~ArW&-nT9=DtQYyj1-pNK#{z1Ku2``a5H`f(__|3V$DgSq43i@m0G0=wy?ABU`+hiLE2pOi>hoTklv zG@QW22}~b*%cwu^`|^5?9)w)%C>QTZ&#QRXUWd=QNb)vRb;2=63`MKqMQXLSFAkBE zAtk=6KBTv2)Z(G=W?pM7z;X>mZ1E!S4UwXL!YbZutDU+~>x9KRT0xWHqM^`dBh zTh;5a0lQk#F3L34?daYq$Pc8UR8LhD%sN?VgJ~BVu;LFU-awP|l`PivzeBaDAXAYA z{{d7T@(#Hac${s}(F%ev6b9h?J;ecUa)dsBmnIchR|VZ2#&*;}PSGVkew$E3>vI3j z|DCZ&S4L3<$MX$~N0DQGIpB#!F4Km#F{9biU|{OBf?8V~$)XtQ327#Yy|B$rc4o97 z8!-U?rg8SEhHJhEq3F>tx2$mdtN&%glr|Q;Frl|(t5$Q|KP}E6<#4^1CAdnfjj*JE zr(5FrmvHAjBuU~{G8U0hy7JumU%d7POSNv4yc~F(ZBV;T12GV+&tGBb8aSm%o2yU+ zibO+^E<%>E_s*-Bwef=n@$cB@zy(CJ^2}H}JJaJt8Ycd5f7~C$)7}1299Ul%B=}9T zm_k;GPmq)mUaGT@VUFyBb;?mpTp1bokiB(MS9xR|QsS8^u=tAU!f5cxly|0xGlAxB z86oYq8`gDa0KY76(7Ix?!W=;)gLFl;bELHmzC(C%5gH&Td~kDs7#o44Y}RFmoOj!i zeU3k=Lg`>wquu-;mJfBdO-0nF=MBI7R0gfCcB;7Xp;SWkU#sTYq>NQ)ooR^adnpBB zKa={TLfGB zDJa-N>C+K&iI;j!B#GMo`|jSSwic-%dgJW4J2P!1bs{9(Kfc6w&o`6!A!y^X(qbQ~ zvHbaVKA+8DcnH3Z=TkTjf_@l42%Kfvg7Fo|jpkk%4boVUj<4b!%T`43Uuyx%flmd9 z&$ZxQBn4<>4w}U~f`D_g>!)};eaolt2P^!>YS^-DLkbs!(`(QYTVcO^s`L_OlM22< z+**`zYN=U<|H3rB7smREGCE303~1wPW#f^(~Odb@g69Y z?WydIV3#VMLM9m2%V4kv(T|Q#OJ(sgY(rj#Vvo@PI=D+A?sBR&VDQ-%4A(712d!UC z>^BxuWsVwqSSX-xKYCMZ6mx&ZgSet5HIYISEG5*&Y?KVX0rZzm2er5bc$~c&Yj4{| z^0W0THZCqDT}77MIB4p)g(A_mP+3w%DRr(52%213gej8c!5crm?Wz zj}|6*^BQIa$+NL>W%cLt{>RqgL5my^lTzZXNSMXM2{)1F)0lV(xedbmfTSTwvM37U zl&pNZ^{#zN>^Qa`$cApNX`FO`#32$VoV%-8mbn-XtPjTA98D*drIVL0NexPVI3>Ne zZ{K!%W4-m2SgZ&~4i5<3MCk(~PyG^u^T~1mgkc?k^D&bA7J*MuBh*i5GhP=lL*E4* z08v7>DGd@Y48*#?^;RGadkFPS(>0Cny@a-roSpe8Q<}X364zNuY;x_TaS{?CY|&oZz+N_Ih(f3(Z3CCs}n8p{-EIsUJvEluia0&&YS?* zNR8#|7awQF^o*^wtn?v*HI@ZK9aWFIs^ZloxHR8u+cxes0U}80Wr3~z-Cd5yBc@)Me(lEGl4A4PGv^)@z|fP^bRqr|7dO}dsNmpU--lbA*^O=yrR zxDXepoer~^kXPVaoC@r059AC|y|mS%&hZ=a4e54{b$Il+Daeb$rTeoP1T7C11$-7$ z`raJkL}g}ajt4dTOKUk9T@|{tuBJ0%VwuzBd|;3_UFP05zIW{$fzn*kA6=2-&bMO{ zQ0kK7ZnsUQ5e7qV z=#R%zx?TYV?`(jg9&#w?FL3(ngB8%^B!0qeuMRsg&>R|M_3byOT;F=An-iDT;bx7jfK>De# zy|=xpb^?eLXWbxZ=r<5ZLg$t?2<3%NRmoW)7EgVXsbzjNEz=kqg9R44GG*#hhf@|F zluz49LgNb47;?e33<_upUX&hsK{GscqVj|tNA@93)$rYlBSiNV?}PAHOr}<@!N>Do zMiW-MGPdnf3@IdTVlK|EK$eMFP*LJY4L-g)fg%KsnK>!(Aymk{2iZN&W*)#K7koKh3E&NDb!A|bpT~3&Ef#8U9F>XQ7%JiaiRJW$)f5iian|Wx4+=ZSCO0{pO29^OMIBPp52auNZT#%Czvg_4Q+oaR! z=sR}{H3BjP(IQBH>8ee7ZE^%fJtqrL0#w3y2i?yk$E3Ws;{cNFKcrY937vw)G-btM zhIv6_VC_H*_1zk*2#1tFjnAdiCU>^Qf|HeiMV zRqhy0m<4Igu_^@(`z`}I_YXN|K%>DOa^PLBd-h;Li~TVp<-7Fu2dEmrD~Za&C$tdLjdSv;)SYqJj; zm+-%ELArte%mdk5D^KTqEVjX+h24ajB>+-zyG1Jh)Ro+}`8qy?Rdo^1Fbl9EXIjJ2 z91cU%`N%Nk^eTk&xorEzpnRrgK+SGn>nuqWL#a8u9^?g!&lQ?1c}YVwm=h?flf{|E zTNBfIKRsWXg_{JtyJCFhpjyZzTvPf2p%w5Nd}~8)-c!xXdpF+Ko#fo^O6=}-@w2DeW+Xatp?L4M%VNr?q##_Mq-+2MJyv!QU6BII*{veDlpo zYmXmXh|IdlN$am%QZ>D3WP_W(C)O7DWMzY< z+ZC6+| z^J-xBw1PNxSwSK=t}grI@f0FqIx?q|)*dW@daD7oHtBBr$2q+98JOS? zO(%?%ol-dZgNfN4D!G2Zu_6%Q`X0yH2dw=Z1Bvasc53hAKPSN)+PNbG(aghm2P01YRYh!V*RbC*VME1Lr6_bc)*Ox7x5o8_&6tz5gasx@FUV7 z>vgO#BxI8_7tHbfs<%qn(AcS6-WA2JKXoJ~4`7lzXXpB9Ho+CaHfhGJekHbc` zk49si@zXIW)_g(Ma!Oe%z~A>C*P~dVU{)2WypQxa4J1%3!*n`8@5m1_~MeYSsbsUWn3 zJA-y)e6qthlHtl#3(x0Cv_qt3BmBs3X}Af#C;v+JS-s{b0QQ56-fS09SlG!~$KSaz z1t<0f5;WmJcJE9)X!^1=#2*UL+z&9xwvq@eqypV@Z1dYzcBPIKobV2S_%k^_x@boK zyg!<-qG3SqJl8vtJCYbq4HHf!3v#~f&-;^wVc-v!X{mg9oF}1pIqr!cDIt5~gnb7Z zKz2=)L*5l^pSUNGxmw}}@k!nw_P9?3G0#V?h9uZxk-eGOG?;%!y8Neprhc@d*gC{q z0b(l9MTehyK}Lncpph8kzZ5gw$6iWnb`sF13?%K1~44s$^(xrr}+uIQX* zgY-;wo7+UW0K-=w+RI#xD;LgKdn*_7t-#E2gv>ChDs>q!r-y#k0lq1`0e%?U@dH_b zyibaY9BZOVujNmTS1Re8@-!*#oB+IIvuiuC@=tC`ttCnpK=hZ52e{n(^I+h|JUj%- z6)3rw2lNsOE-&_lQ$I)??8dNR1$9b7&_d(PD}mC+XIl((QCo;tE|Xuy zWy);EWt%GFf~FtAC4Z6z9>OH~I=l+j>YK+C@ljV@#!UB&WqMN(6>C`j;O*nuyIJ#` zfk~{iiNcI5w{Ianv@O`ma_FMfS1^LVP_aVjsJu5J84F5d%!pl?kw zp9gS0;fKKu$^4W2;E35xR3Op?aOU&)MP3>Ze+frfBrcO3$Az*5XHNJVVX98>O;efc zzylpE^Yx;Fz3KrCAr(30Qi!SUW1IG^N48wu>1S;4Gxzt!+Y(ncP~DCBq(8QPHmCDN zOa4$|{howzDo!nYPdkr@aDWka-yn>3fS;41ud2;Uxs}nC`&`DS;8?pE9ggM}ejA9N zft}B%%b8^wm(Yie2Bz|{#vq4KEi-1I`Bo)Q~zB3%sk|jGSvI{J* zkVMYMn>TO9xzUv=v4TIoK7M)HK3(0=gEn}4zPew-BkY~(YS11ObXJ>y4uXYH(;M5V ze|rQUTwjFVA*}DO-AB@%mzQIg9c_Afvl)T`8!HM`E%9Q(@ayhXf7*KIo=_ zLiHZ002wT*#t!$|fbB!qhoH?K8mgm*%DQ-NG{u(W&1-<5+hk+Q&s)AkxRRejHH~1& z7oMwhs*k}=kC10_lC5aJ6*pVrk@hu~bk-qJ!+7{*bK;!ubLdO|F-ggVAO~z3-FfnO zdwq9zcfE>MEND~ef}snw3Xz*GXiFee6KLpGtGy>g78OdlTqw{cS6}jhwfsNU?JtS% zm=4Dt@m4S;w2nn+Rq*3c@e?mlJK6ZbcUb636QJaG*H?D{&w;}?R%(W>)h04-qz9YG zHTs%S1}y+2`$f~2D9(%CIW#$?O{DV!So-7!?W3hj)sPyJaeP%pXbxaaA?czj>PV#V z_L3z6J|(-b{~r8;2CV&V0X>ymm1QP7r8?Gq1fTwS-RefUvs@#XpX`7y=r*7u-uqrT zMIuK)2g_y*e}&F*^h*?@Z!VVc0adxl=uf_qRk8CLqLTzU;U>`I@T2?WM zPdQA(giT4bOl64?SPmbUD$*24MPJ6)ssq8rmEOgGQk>koghza2Ku+V7mdq4sk%ma7 z2-{Fy=PVJCMelk}lDnIao&e*W{@=g*r-R}sL0=IA)l#7-O;ea9~%G*QkH!KG&?>Oi%Z zcVm?zH7!Q{Om5UsMEP)%PZBb*rF`n;d56Tz#p?Uh{Jm7I!DFkxk@k-cpWMFH;2(?s z$Ur%r#aL7`d{)p>Z)Y9^;iR8~M1 z&NS<4m6z)@EoRpxecEBpHcR^YKSz4f{^?-PluIqpqW5Zl1fA7b+9+}ObniY7hA#bV~)WHj`K2!I+uaA*5(`c$}3~ zO^=%}5Iu)qVJVk3;?UjW=ENqnsk94GN~PVC#gYN65*yhj&9=&a@7M%_366upVQN z@%pvHf~cqCZ5W}=F6@J>p5kwz3q^6#+wXkuyc|}Q39jL!`F$F=gW1@V$Xf zSv4|fOXv*D4wOO}E>p*C&Q7!fUg&aSoL31eyhf<>!`%>M{0LIJn9`|k6cxOBcL|;z zU;=LVA=Oz)|3YV?;iC*ch1}D&LYKf+VDCa{RZ$z?z|L#%_%#Z4uXG)L4!%Dnvrm(3 z6h8;aYBRf=WX(04-4C+0^L@AGP}f)m?GBA0K8(-a0vDf*?zs!JPliw07sDr$55~u(lR6WJ znpNQBR+Gs`j=6;Y^DaEFhkpIx25!=7~qmxgr=UwoEDE135yf;VfSnmA+fCeFZ zu}FBFjZ?vH(?AeCXTM^k2r0FkwrVLzL?CehrBdkykhocEdmK;W?3&%#27~x_X4kkP zL@iwG^=RIkH}9?I)R-C@Sik+aF4k%;XEHc_s`?M-~)}_jdh^A4uVJLb0`DD#@ag7xh1pl z5x#Fx1`jsCUK<9sa2Gto7O@9*y4r$mKto4Tl+y^zL7O_FTYWFeN8VAWjq|`7>BLfGVZx*-x&b{cp_gIncVoJ8sY+dAt z@x&Fe1~lvlJxC?qjO2ZkdZs+PP;u%jk`+bl9pHzW1>d|; z(MPywT`qnJ_OVf!c1{l?A%2tP42)wsYqy~Eh3yfMkgO0aOx7 zIxXMCIN&v$uO2GK9e$@Uq;&iF=0-A6YL2F+M*RY=V}#7QssVVMtycSQ+cpsXIsPjS z4N%xZ5_?#J4NIHCi#v#3HgY{?QP#cw%3Bn5{)f`XeFE(VaR9R$gew@rNcEu>|Ba1g6 ztSe?^${1@|fs5P)+;-G}2!29H(4lfLlNJ-m4Y)ainvVK#kA>AF&?zj+%;{rSNMlnh zKwGvgt(;{DnoYC?y;x?rK;xuiD`^s_((}vqaWTs?kV@r6qRn0Qc)}nPZ)(Bh4s5%=3OQTxNpvx+) zHSnIaATs#DZ6!~XLJYw9*710nVL3zTKT$YX=GFu&YP3sA6Dm~tmr47f(2mltVzPW@ z$qg0;D*KexL~vCFyL81-y^NGGy{*2@0vEF*2#O&d%4-MGJFPKQZYY@l71@+o_Jz1y ztudE9(p2Fkj+TO=ol;%s!Lq`kR z{7n?Q6KU{-#EcgIt2gG7Jp*7nh-bK$Ng}*ry$KuV=E&Ew8S#33o?1Kh*5PumlbVq> zmHo;099?A?Mm3*s{@;NPJWQ0= zUsXLC(7OdC+403{FdTvZ-k2S&t)2IZDLOT$<+LIszqb+J@Qz^BUMcsMr=8G`d?^SQ zQO@r#(ISl8xg~gyM(fX;Ki`pd8~Sh&1X z|6@r#=I7t70(sq%Y{S@Ylkwgf+3u{*bm)VT9%G)M1}Rk&IirgG16;sA$_xd5*O?;K zx`8Yie}TC5FClWtKE4-toOO=R3W7io#_vAGf)|zEpj#1C1ReCgY&vUGIUBR1;oE1X zgp#+JkMGYfL`wx&bvmS(K5ve@ZIs-`VQAxc18*rB!+2g-}_;&YeEuW@PbrM8TuuabS?9-p?E~^YD1s`ZSV#cIET`(6L_3ukHHRt zFbsyz?kSpZQ37)C=$YsNFTTJsMkh1ebCa zAJDdx&Lb#I2nPKN&rwz}mT6@>RM0g1Q9o}rzq2g%+Tt%_+6Exvi}PqqJ1T{q2gp4R zBLyBo!48m0Iy(fAz3`B~Md(*a1`TfL=eke8@BH<)jq`f%wsQ={)+Zxj8 zllEa0#Mk80HGYn^(~5lo_L5&*xg>a;g^|H(!!Qhn&(Twu=F)|*^C0K49SVc7yK!xK z(MaTAJMCcX-KR7yG;POxF_!;N`X!a**`Zh4*Ju6yw%O%DSIz-B-u{YG|7LCK0?p9b@W7A^|MNz2fEKH|y z_lZ_o{No?Lax~{sm#S$<6(c(Rw3E$Qu{DTOD3N6^@C$(j$aA#~c${s`!3x4K3ib2I|T)A`9t#k9R>Cn^f+9u<8ZTQ)k!4AuZ<$E zx*&L)fMSAtSZmo_)fwt+Bdt{%+4kBkEVG-TXcH(RL&E-@O~Jfan5KgF zU^(;>+9ub6@!#8Oh#+uVzxVF-U5vG^pysE%*yp0icbDRPyyoJ-VvkY7jKWYmu7K_! z2xLnZJ4mdfRyr-nQ3p5Ca`^0zaUve`J%p!cQT0M;3@ z3xB2rwr$*MJFM;q!1lGn!*_$S3K1jtaZhN7Hag&iGZs!5g$#O)_><6m3yPYI5(<=| zSJD}*f;R?TVa?w7tBw=Ag8_J)rB+Q(+dvRK<6kinm#~VG0*6ZE1Qk#!Br2Li5kl6+ zUdLPP-Dr1Bl2-ip&a6LT8wVZ|2RLoz0<4R4fv>yZCnbWgdPF4)L2*VrK}f zGezV;FMf&0g_*Pt{saddD=_IqKI>UaU#7&y&4SliAIXTs5>lb8Tq+SmqzmQxAi&V? z@#^;SXLx>pfBx_^gDHbt8v$%OfXa>G+YYQ$4wRYoEIi9}^rm7T9SY@)$JoR2Akqq% zMJYE(kLQAH4SKF)GQ0vS!)!^uZtVQw`(=1>Gyeeqr;YXM_WZiCo_r31DGPwXJQa0n zw_OR47OX?&!~jlVA)TEB)P`=EZSpH_EI!WQxZQwaz~(cY?aFJvOESK+__C?8{nEF= zru_Kvickn$iDZ;ynxQgo3Mv113!HuF|o`F``7TB9^vrOyfVpT(w<#wCHs5`B~ zsVrcT5!{In9xO7N| zCDwa#2vy|uzVL^p)TZ&xG%~XCccWJcS?b6tmhTC;mM(>rMn5N!$`-^X64pX4aP4i& z&4noOpn}yj7V25)wb~c${^Pu?oUK5JcK zfdP1&byeGL+c*$?j=o~xyx4-1*p{0-7=7qw(>7>P7jCv#V0R(VB4x9cNQ0yj8*Tr+ zGo-9boCSg;9?qOObLQf6nJX@G`26PfL;5~C$N!}g$B)suR9R7R0Rz4MCNg)q9YoPp z96=19S@vCU2zJ;C#)U!qRUhNa{ku1Jd0Z$Z-xfS<8 zg!9OXN`Nd1v0();d|_*i$Og-oYJ+}6wh))-9LQrb{2_bBtbn!1ScQMdpsI3>IkA?| zVf0kMyq<(X@flM}I}rO!;2=!L^JF@{zMhOHlL-+QSnJ_9AAznSN9Ldq%sPTdTFE$b z%#)_<8e^&OW4dR#=I~Db;T7|BH<}_0Mc3+697jc*4w!jv1$9G~(jf?%lNk!?EC~X) zNXirk@7i>rPH018O176}RUY;G>g&U@`i;Q`VAHIIOqV-mM6Ejw$-FK%Fd_^k_N^He zZt1uE^OM0~ptQOuHOios%!1c|FoZGMfDE-;k!i(iNR-`hYOyU*&EpJq_)vJ`cnM)7 zLa)|H+|WFQfSu7hZBe$o^1}G^6ds108ZP<1-KiF3gY(zSKl#-Cx;(uW_kaG~yBK+X zGoR3nfarE>qonXvs$tPC82>al5?T39#XzU{Vs}dK@2MeGO!ehdJChV zS!{nGh!yf+RpHv@|6flWBsBOjTD*J>LkBxIv3rdO;)RRUQ*ZI?p&`|to?EODhki%5 zquEjjiAev-Hwy>j$>6yAto7oa(xJiR64blyNNf+ z3deT6>p+C~?>DpSm$hT=Xrw?g-g)-fd3kp7N-UFzFX3hsO#N_jGrO9Mr{U$`6&gjt z_v{0n6dTA`E_s+{Dpb3S!{gxKWC$|j3&CO_lZ4^P;J~tlr z$?c7&ArMidiG+Kh){vyY@>Q|niE6=}U0Kd&lgYI|8he1=;nnzK0QYGTN3dpF4s*^E zVE9ido#%v`ihIOpNPriQ>o{y%N^oGCSt!F=-&L%hEXcmqU*=&GI-t}UeQ}k_t zZ%hU3fsZHJY!F-av2X0Zx9pc!=jZ;I?0*;rQsu=$mGL_rP&H72i9}hv7ISBT(N)k| z4MPu%d9h|W3^DAUb}SOfm5bmf!vPH82Z8>0|DHYrtkXD>pw?Vh9;8eC*&uqbOAFW( z2&pTACD0Tl#0>2sp*6X|VtLMXuko|qMy}Ibg;Y27W&};-*n&z*rBFyjl~`qZadi;# z@!;s&n_Xx}qz zkk{~u^9+Qla~Nz$y3g=R0<^6gnG^6tQ8OwPBLS7Ws1t8ctm{e*GyR% zhA8FowekB*c~1BiGlQqoL2~%Rf@cbrX>MiSEgftHqD)G2pfYH*n+~JiZk4A+hJ;we z^P0A9teW?R$vX;9fB%(xc3bd0NVcPm%VX8Cts|SDs5(h%4p9o8bh~J}aob=`ZWO!2 z!&!>zX|PI{p#>s>4yYE|x#G<%X+rm+BZ( z%8EXtP!Nkd4ZhPoNh?pe z!%Ts52 zdHNtCRBYh1FniZG2;aa+nn&1*4!TRrE0sJ4EO`m zNZOOB=1CS@xM)wKq-k4k(04iMSMNYuLjJFt*`oZSuS5j;`rVAbS}ke(Eg6p|fj_$q z#-R0AuMX)tst?eEIjlhcgUT-)$PDAh^a7b~XzW&8UqDS7_0wnK9J2X5=i3+l5YS=3 zE&GvLb^R-Ftqusk_`%s^>N{JOyU(^|KIMArjbF#@InR5B>>*+i@SOgpi_qcbpfA#R@GzmJZP9>@4uVlsNr8jjelV z@?yaev{sTsnlAoKK0qW^Lh0WSk&yV~Unu4 zOYwm3nA6D7D(o!B(>=5^OAB>kc@l}`;9m_?68*X!c$}S)F$=;l5QS&+D-OD75Gt;j zLy0}CtIJB4g9O?w( z2waZAU@d#P1mbqoYS^A zpWjxzoDU3g48KKXCj1#C66E>~A(ijn2Qn3Rd$y>`vN(90l~X-y!!Qt?qhE30#W9A^ zHJ-Yq(4o+D7Q&WmQAHBcskD^-_nz!Db?lfl%Yp9Wy?5{IOtq$wEj-=tcl`0;dCzw& zql7Z@EXM-+SR*)r9oH%#S11qg#a4AEJQOKkvDLd(4HOZTuGn`q8)DRd8K_7x5JEIT zy&BnoG6=?(Ahfpiv_K$DbUcBC?Oo5sdT!^b%unEXZB0NJG@|^#5{ArF1#Pgdju0nh zwU)x?^VSs0UPtSt)(z`jY|#>yhNA;jo2b!xF$BStmlT zJ6{qwu#Oi>#r53fqobp@3DTc?k&5L4$$ucVwQteg0(s3Iy6@DWt15(Z(locI=r5dz zM&HO{b2`wkEPmlqH{6J2BviFPCX9ZowB!$-EvH zW;HbBG`2+t^4;A(sfKZrG`J-s0XNzRJ7MC(nhcV%Cwr%Z%VLk@}0p`4J)AtjsYGy=&IUQ7Kpx-{-JY_9rOgTXVY zA3;5H?zkU#oQ0504uUWcM%VTfldw_34Oko9Y2*M6p;Ksr)21_+czYWRNYucr`t!Zl znP$$CK?~R0<#an%l|HL1?H%FrNU_AB5eU(@5t&P+9^M%+jzeERwRF)BZ5IaFe-vi>Ik_w>2bkN4Qx2H!PMHe zsZhIA@UHnPU~G^?OHy^dHw$X?ruWAmWbt8T{CWPrd41=0g(+a!e99(?%8-n+>IDQ` zwmZ3t0eGC9RoiYGHxPZczG4z1uq7x?``Q$JOOOCYiv-DAfCaT2Y6)}6HOaNK{ra5Y zuHuy}q;UmH=4CEt&K#1L)_Bc^K7IY?%e(5s;t~_<`Nw>5X?^W_&Gcu=8k;xGU&m`U z$kO7?iaxOGC2s_h)F$r>6$Pb!K{5}y)0a_U`{^3#wqx7SNj38_nAaoe}-^B+^Z z)I}gR*LADc^wNZ|QITFH{JmULx~~d?Yk2*(Jf8o)Tuvc&p?8`#OsW%pqg9!2h(4N; z+5-fXxeWy(+M9okkH3ga~3CxQyi=h1F$eyy!BuXAMnRnTweapD+A zkUVj3LdtV@n2z2O;BVVno9OxeoMB< za%Vw&Dev97VJjY44?W?%AUE*cWS_xhkVoKXPweF?l(QG~-fcTsf-2O6BsT8WdO`x*^GPs=s1 z1{ZmvJ{1Xly}gwaEHqi)VN(a#`;f_na7WJG@p0H$Pb9#ULXX5w^-U$;SDGi;mW3I6 zU>c~$`s*s?!ST;281KpxTG;FEk=2Qw2dW}233+bEOWi5I?bQ}~f0S;aRi5W(T?$9eLvDiq-|o47Uh2gqPJuOx?~>c| zD8C1IoKwzB%S%a3QwVZ(boOv{bn6+GD zT*|3=DVb?p0D9*t?!Om!oSje03c@fDJU3sl(2IutfhP|>z=Gh#V+hTrUD|FWiPq1z ztrn#R1urw~ursrpl4XMyPMd9h+^O?AKd8M(dx;ols=#|h&v0{OfS984Xd73mPB--; zM1L(Jd`ODeIw@hG;=5_Q z)9Mj6vrB_=|<2V$4 zroZBdc^Ig=P}oPFP@7p01$N8Kj#jHxWRjEC!nG^g;WDH7?|Y7&HfcgT>aK(!cFyHH zx9@}xvd}peFik!^q*uKU^ei>5jy>mXnR&2Sq{?JZ@V$4C&4qWQvP?c_{j4nfPPJauz%_V@YlA9^YEN+GXKg|Qi?FIIZ8`QxJ31ejhw zJp6S_4=_Ay-T!_$x=F?`Jn!{R`#tExNGRpN&k@`bGg(MNP?S3J(r9)9D6%<+2?1-V zy;LAMa9>*OKpXuHtr@T_q4V`UTYMI(M0#=B^KVP!WHNF#81hl*qs(aMVVyY6&;a7) zW)i8PWR&b3Ryhx?s5kX1kDR+-=n;{<4JGwwB)fq57tvvvjbuCPU|d-pBH13I!QuuWZ$kSs6@Z)J&M4z`a&)fHf+S=L{ygZUG_( zQ7vF+D>ZxqAz&t-Q3n}Dw5fd(x~s}vNDH~Dk^T4nsw)z#b|U6mE3R|u&}izm>p!*4TEKM)2V zIkt8xwxnUjJ1v;hEe!C8d*tI)aykA8Jkp`M$0uBaCe7U4j}!WPFnlHc*@=z}$F8QT zmP?vs+!NMnkdbjID*Tz8k@& zQ#P2H#e(>e4RXp@C^}u&wB;bk|J=hAsRv!0-0;?mCz^}L(h~=J;7#CBZ*)Xx-BPdm zV7L9RyhTY}2)y~mt)_vh-b=V?z{M%Xsc&q{BKqG4ry^KJ7?)?;|I zNRh~vqIpKk@o=1|y_M*;txXb1Nrbm&dnO{n)~)WQw}m`_YfG$mOnXgM{=4by0)%>5 z)q8s1?~YLV|MBUuW!sb4ZA*1UWuvQPbdxfneY&6CUVcvRZ(z9nZ8DwSPvPj7v*R5W z?vhFJXEGtN^BrPeuP5Wt<#bG9zaDRp#T+Y6r&t!PruM?l(60u^B5ykgrKw16*jy)_ zNn!EM{vw`jkZ{#e_|mzfbT)?8qBH6_(udTbF>Le0nedz|Jy|N%IvJ7f10L=a>JS|ni~7mM_F^EqT<}*BwO~p=Xf1q)+lSw%Y+T&| zjIWdR^B-TXlh4%&e#AM6E3AV$ntUI3@8IC4q}bhk)ZVPw_#A!*yV3MJG&Jg57C?Lx z|9jzB{{=qbT#d8C0eGB^R!wi)I1oMCzhZzau(sf)$sX73p(v8=0!;(By$g&qw1`Ng zKvH&%MgRNGkd!US_HJ@WqWOF?^El*(W)JJyKKn|IQX z*FI>#XsEEW%6V15g4!;++Ao7c2qo&hIJh&vVDQJlV4Upz5GxB4c=xK>JSEAywiF(d zrcu@!J8hW70@V&%QM|6lny#&lD5K(7y)+gI8gQZsS63erp6FJ2-{!3~KM(8qul1Bx zS^S%C(J!X_T9UOtw8%VVz$n5Pg1yv>i-A_KGpZ!&ShXYn&=O`4vGsCTUgQ@Yn@&5m)mSo+0^v`%Uk`+>@E)z6iW@<5RC*ibeJ`D53djFT z=VL?hX9y6nOzBNbIFWw<4zj>w)-8@nx#>0Pcn!8Z>dUsPFn(DRtJ#gHN6dS)NObcR0&1@;0_OsusilaPh$w-IX-MgA z=gh}2WJzA3A&V<`EIL!~uqTK+G~A*yKtlm1KH=JsH6xRM>x_UDA5~qBJ2j zOc2&&Tq>wY`PPt%E*!Gt&!FmMDN&!42u-WsaflbB6@(QfU4_hhRo5g0hyD$bBD@d` zge3i5*s@30DCejbxp`%Xo$IK<2auw6v?gCV5BrK*yhjjJmMBYJZ7){nqPqQ!*UoK& zc8eTId5-W*abosp#WHPZZhkl&I#~@>6weTkm9CM*YC0wtmq|F)+~w7kjoH&=J|k9* z2^Yu)@$T%)ovuDcaLV#F2;(a+tv9+@xzcUmW`E&W6s0@zh+zKifR0L-CQUJ_4E>_F zh{C#br`Gg0p3-IRE#2aT|JuEFr6+@0(^M|L3McOgdFic~&?JY8FwUNGfg)!v31Zat zPn&3IN1y58_Uo*OEdIUv2H_CNgx>=(8W~D;Y~VOjdUzzCXdm|6X`S~DkxqKdgvbJT z_ zvNBaxCSJigwL?r&dLgayj+{@~4KC~`xFCLlL&~gKQX#C={`hpq`^hNz zOP-rVc$t!L2#u#F*RLHCC)-CRzO+cfFC;x7ym-iEWh`|%s6&Ml>DawmXH@^#&`Oau z4Cz58W(kW>+AnoYo5&l48VT(AStr0JE>z!)j&u&EekcP(?NvC|PUf&o0Ftz1GH3|v zV^}SPPTR+{EV=Qyak5U169tx6Ls#32%)?Z9fQ~?BS>%$z=45#qINcs+#t#9z4?8XU z4I_aACp}t*8Pi(kED&6`S4o*;?5>8dw@+*^yBc=i0Xv(l!Ft7H4<`z}3fXGo$(2Q6 zEGpfC(S?dsfQOW;a6=cKcVX?4TuF^-2l7tV9)Laq80!K+y{M(obEOB>4f3h)pfqT% zB=%@WQr&l?=1ncNQ;*yJUsS11XAo&>3 zJe<^}hC0@^RhZasduH0+*a{QzE0}Nm4hRnUhx6(EJf4RzAZPltn{d2}P2t;n)`OQ&LDi#?TYN)XrvDO7OJ?mtFGEM37XX377zE&zQAV$6@J;DedV%20D*qlpe zUt`bViX6I0UOE;&NdF|IW;a{zSv6F~`L7d25G{G` zg#{^@O;Rr!>?J|lP$$!qz#%6}Ps%S)2%|1-hIcdyW7$^7s*UxM1w{DMq5d@Bb=!J_ zW@$(;5G0UhYJ<|tYw(Pasw)#tvc$xmqw3p1he(#kD@YPg^vKoElB2J}B_vA)85s7+aq?O{xkNwj1Sk!llVlZ>dE`cuA`)`WIX-Xuq-* zc${sI!D_=W42I9qQy6mDLN2{Fr?L$Sg)-O!7|&K3vsNyWT_~mRUONpXf$1RQPyYS? zs(numd-e0R|J2``kDcB^oq;D@Fp$W}MkPp2pN;*~h7Y1{P}x8dl^qPLhS8$GV3~JN zB@%xibn1fq8wA&@7OHCe!5ojX+sQKreu6%*MV-#h#7U3@bsRj6wol~vsyWem)xo_3 z@H7kd_J7i*^sH55%|kns*s4jzcnT5ACgps+FAl_dK;xucsGE_!=M;4|^gTQP$w6%7 zybXAqUCloW!axiK@VWOX0=H=C(%H$apin`aL+#~ku;%10iuKc5vG{Mx`|)_O?8T9X zZM7_MUDvDK1}kGnVxd1ZbW@4~EDEV>N+i)Z;5npO8zd=Q13SpI+4oeKEXx2A-w73{ zrxQN!Up~@+!TUry`9%^RgdFst_=JwXg2bu&dinOYYa-<1;(TE&iDPdbdr(GJx*B+# zb&tVr!Y~Ym&-GJ8^fC}k+OB;7I7~tsyGNl(S4)i(r3nln-kn5*s-o+~vj2Sf{kz15 zq-xk>Vj-36)G!V2GRI_&*>y8!6D3$|c${U7QES356oudOD=ze9jtaHVz3eHonaZjuS%tlY*xX`}#*su3 zfBjm@YQeXA&iU>anZ{~t;Cqv^54y|cnnYbT7N^y7R|;P7B`+isf(w|Z&xl}KUa@~* z@sQ-~lkSU3mUQ{WD!HXUut<_*+zTpJY`VT>S(*@HYygM#ReGcGpsaqW&(;U1kIKPo zi!tJsvY9_5{3hvo4ph7%^hU`k5)fC{od$->x&qsX03vXVf?%g+^?!n7-Xw% zot8wM&otx@f9+{%w2lIJoQ+s*Z`(Ey{#^fxTk;`x;<#z{EonP+TZRVPnxV;nVqM|U z5@ivUNR6cIdPV>Ho+Bk&w&k=!QBx%E?z!h)Naa{%I+YpSKfL|%&tImm2gi6*T5cu= zPfrJQO84H*>g>c(A~ek;)lQPqR0_SKrC7Nu5{?ATR$h{@R;Rli$MQnjtT0P*B9qu=m7co^);H!H<F#lAvHks;rFlSYTrx zrK|!hrPHfM)w2mCB%Qy6LMmlBla>^)77QB>6opC7Wjdx@>LdW!bu@)))`VwPt0JYu zlvN==wVgLrN|<9KuTK!uz`0{9bC ztvYBcqqGNH0jQHk>YoU3no2K{{Df7ofT^SvUKt(fiK`(Z0UKr3l%&9(%VkkCx|X>r zWFQT01T|rm7NSroEv*j{8|qu|mj@{Agg?UI@ZqNb10LH8sp*WKg>A0E5cn5frW1OA zPWcIG67@i1Lw=h?bn^Gb$vB8CTf_lGqpN`eff?_PZv1o}!!)6rkT8)y%4(eizNB_F zl>1z@kk$?}gvv@M?E?5Vk$Ypjj1H=>=k8awc~li*bwo?$^MI^jK2=sG9`RQhtxQcz zV?RQ!Q5CcxyjL6TO@QatEQfA3ru)0!Z>IORf8C6(n;m#2lL@_+R*r)yIBwutWZrpU zJ<|itZWj0fXF4U5ArgA-^KCVaJ+Tm~fWc9$j)K8j7bO`tS6suC@ZX@J4f`7YWZOR{ zJ)-}-#V7mF1Ah>e>TjL zGYprT5OZnjdCrkiF?nMurABYM9ip5_<|R#=9Bi{sIma`v{hx$`WsJvxFf>GSLq0UcVw=@Dg;KV&nDT%k^OQ^tyd>`#|)WF2?+C+Z27_qwSoZ)>ff#5GAVwp$hYG zpn)!ILMuE&l?Rp2Uh7GUn$SPk5zun zZGw{++&a)8b_3Iw+BjOv!-~E_mXNarN+C`6 zD}jAvG)bGbTYNtoby#fJJv(DAll_G*#r+K;#O+W4R|ba%D8`YsU7UxSM~R#AjW?v4llGb}g)9EKYvG=P= z^7mXbJ3N)2DqP^EnFe|EHEzPyCO1VYZF_QrUhwHaZbs!_)0!Du3LHHhD3s}a1A{4_ zVph#+FQ?PtFv<;a@6bLQUyi>YjYbD}>W#-1h7tg9RqZ6;_Q8_jiq(^Z?XmgG?Qb`s zKcTq7T@5{)B`Cvr6fViQ81l6nm(DQHA1^xG8#i_4woUe1$v84@9n27J$)<;DfXZIpRGTYHKftN>(g`hBC1?k!nqTb zCFPPv*!cY$Dt;I?Lf|K3AW)tngBoOjnq}YdVZv63$RR0)tiVa!&E45OAMLGN>^fxclP%Ua14p};8F$(X6B`0h;yY@l%y8rDP-oADCFnm z0F}fi7nBxjl;vlpXj*e|DW~S8WTq+TYoY5@(9-7u0AB|oS+$MV(}5MSk9FXR(XP{eRA3NSrnC~qu8Z^bi_VM0a&B3<)wT!D(&I|~nqNC`jz~xoWjW&`GWyIZoqhYca;u?Lpa`~yc{LxF`R5tUnIZMWO@sqHyQl6PLfdqUpMLR zFKsHsi9)EImeG6UZ5D;%TuRWS~w)zaYG#_ z60Z?%64!!Wj)ue0&&lBX9dvV5Rch}1JbXmn^LdN6T`jPoSr3zzhc8?!P^i{(VfYb@ zHPQk$z#+^qLt2DsvKdxx?FH~)N5qq}qm<=VEIE2R9j<5SwFJEyz%k=?VkD00|A!qv zvhvEdr>!Hp<1W0***ANq-@3D~xC;KdFYy}bs5o`Az9W&7A7sqwF^c=d-!@@#)Duo`6B99XhM>3Kex-I+PJ0m%czd~Ry zaxycTdGC#8)~eDaR`BcR%S-Y5+mG+XYqBPT(m1ju&f8Xc(Bg~*YH#1*U$T-$I}fdP z>Y%X%c@!2hcNAZCS6?(W!XL@h2P*C1Y)afE@4G&=w}?fFPTn$Kj4C19w^fCf?$0rV zbDG(yzkO!ml_X=$d=}A$sv%8*f4mjar!zbmpE(Q}1xig0qrw?;YBmYskACb{9A)OpNuojdZ!e1y%+LS_)W?#J(cS$+-Yhnm5t^F zM&Z+YMj!&+s8eTUV(waZPS)Sc!{rPI^y(@1YRs7=uWBW| zqre6AprSjg*h+fQBCZY2ctAgV-ixgpuY`Q=?hPy5!z{}C=Z;DZl^l{<{W-lCWW_xt>78vhf;d~JCU;}aN_PJ{)e46ex9ep{_ zk8X5mDXmeLs!IL?thr|jyft{7eN#(sgD?y}lV4%AmlmN`J@+(iNZV~oMR;(+AQgwFq{N$|2lzMS~mA!Wj4Z6jCBM<77?72I0qY} zxYOktUCTr?iu~|U491lI1Ls71@ zUElx1A2dwjo4d3ZGKp_rlBb8)7*@iG1pU#(SNgt zBeK6IuG3#1%fHjcF5-NVkqCkS{4<2#NP|5nIIBV$Cc3b(Tf{M?u$G*a!^H8ZP|!7s z7l>W$(n>B(zjU2+H!0MI#rsC{=FRidl4@tM)<5pQ+)W!I6Ri2)WG5QEksY&{`T2yjUhOAB0#nyMN1;4?# zjl@{KtUAxo1@_jkNTl{+-BK)1k+(h}bh3=-K|4G0KyDb{noz1gXY?QT6!D!))Ht-g zdl3`zhj-Z?ouA&z$^R~7WfMTA;SRiO_Y=^05l0&`EzAjC;A!3{vLE{8=3`9Lh_pnT zDVQE9hz~uLo#2gp=uA6vCPc~CPQbiq@DiLu1jJ|V@lhwsIPP{Oa=E3sB;*c~ah8`$S*Cp$h>SAiw z@{kkUF-F(NV7lk(7!Y@@M~Ykki>cBD|I1a%+`N|Z*6fD}oVlTFBpt_bXLyCNC*1ni z=;Xq`jjO*s|AooBSC@Pfms+B;RI9(aSk$w$B6ysYQ9WzJFc97KD-OIg#t=xCK*mDb zKya3%t5D>#b2L61=}g-G`btXc#Ld)2x{r77J)JVDxxyM=);qo5JumgE%0f_FivjUl`E2!`4u+=X5_?sU#Wa&Q!_qpM&I+=449nL#}OP+f~8+J&sL_0hfTi z3#UFi(}#uV$Z$tlq?bo|p2Ukwazc(|+0Ww3tzJKE%Qtl)aF~3c_7aqu2{~~Aw|O_n z=z7{A9veX`!;{DHTmLGU;!xNbl>5ylAi1Ji{oa9$8&uzvro5ZIM|hl#R84Q&Fbq9M zzk+Bkju52SIj0^rY*?o)f9cxo0L!#0ZMgRLKe{{B+6}#BrBl#ZrBztQcZ?J(M zclX~vKd7%_%aQf?7Z6+PYnKheO~i8`JhV4LB#OBvc+&0{MtBjMORxr{(WH-JV?9A7 z8>G5b5u07LESGy20ZxJ}6~~dGk{ySo@k&$n$(}qKs9ViIYW1hxQtkhnMzz|Z_PKl4 zH}Rg36h*{Y*xE!PKF<*C+$_3ad<+TQydq*97(a>uszZ#qBNZNhVojZ&M@ zz<%PrDz?`8*1#JCHbE*K)M9_*8tc&OGXjO7s?Q9(aM3Q8%%KdnVp1-gltI$gr8ib> zj3pbqX8Z6rJ7w8O-F3)=v44{&w;onz_E&a&YWCc@J$$gZ*lhl>`_hxEQx|*YZuc%< zy*H(DmFRCjQKtr=nycmLW?yyrPm~Y4ybIA5y|E3jozs7%Sn&rZvxeKWFL<1FQqgW2 zF%W!?zhWg`k`qck;HpX&AfakSD{?50ma+G^RSx@PuMeuK{~mMX8lcI8CC_+vyq@)l zDr>NUr?MzlPjW6s36aHjLX61jCK`kS6ffdI9f5g>uDTl&Y zF`X^Pr_}1#v?yoOGj1|37LOO)YEdqW$@dF(@$-IJEGF}@_?rt6*hkH9b~#xF3X&`_ zsU{a!x>g>twO3Zx2k2~IIFcSK^k_Ba$GQ&v+JynBEn4%okCwgCFF0W9cFNlX?lLHA z3<%0&%3|9_Mgt~#vKwf<(-^{G>nh+Hfvlm9OjHd7)J&-v$nK7wsd`JS*3Hns&iNNZ zUZ%yrFSyf&PM?%6z`d-fH+lH>V+L8(UTE6%ZrbO4ga^+J4FSp|t7vn5s_v8@!#UAJ z97E^GK&IC>xxhW6x6t?Z&ngC#{pa}P++(AdI+U&AI^?4Fdp}N1wkgx=|7M(pn-9dv z`xB`cr7=tA6tONb&6tep_Fq6-6JV6&X$ z+*rhy$RAY;;=glF+$u#SLlVDt_nzO`nO4|<;vdi3%|>ir-`8)O$EUS;VHv?1;R?et zjdd3q;LAg!mAV_>x9yTK=WVDx9{~YId*QtFuwu*~2Z%=dUN<~1W)*dhE*KCBZ}O2X zN?y0p#&1BUcL)vddfeTt{x#s=Y;jeRr{5J@Oqv#q)D+nga-q)8jBxsk8h*%5-=J>r+Ln`W9%rD0a82r1>@JEXpwMKmy0)Z#7BX6oB<#J z-~XQypAxW>2o#vi;jlQV2)@M}%`bi&A7qTQDWQ|62zcsc4jgyb0GOF1L<-OAdy2-D tFDyW&#L@M5oLj;&mt})2695Vb0vUd~@0o~~*#qY0wst)_Zp7SMgj;>}a0>tc literal 0 HcmV?d00001 diff --git a/test/test_pack.py b/test/test_pack.py index a0c8464e0..199b6c61c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -13,14 +13,13 @@ class TestPack(TestBase): - def test_pack_index(self): - # read v2 index information - index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') - index = PackIndex(index_file) - + packindexfile_v2 = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') + packindexfile_v1 = fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx') + + def _assert_index_file(self, index, version, size): assert index.packfile_checksum != index.indexfile_checksum - assert index.version == 2 - assert index.size == 30 + assert index.version == version + assert index.size == size # get all data of all objects for oidx in xrange(index.size): @@ -35,4 +34,14 @@ def test_pack_index(self): assert entry[2] == index.crc(oidx) # END for each object index in indexfile + + def test_pack_index(self): + # check version 1 and 2 + index = PackIndex(self.packindexfile_v1) + self._assert_index_file(index, 1, 67) + + index = PackIndex(self.packindexfile_v2) + self._assert_index_file(index, 2, 30) + + From 3b902ed6bf75bb04bdf5703a564f539d8d2e43d8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 23:35:46 +0200 Subject: [PATCH 0014/3719] Initial version of a pack design that should be able to solve the problem nicely streams: added pack specific Info and Stream types, including test --- fun.py | 27 ++++---- pack.py | 147 ++++++++++++++++++++++++++++++++++++++++---- stream.py | 84 ++++++++++++++++++++++++- test/test_pack.py | 54 ++++++++++++---- test/test_stream.py | 28 +++++++++ 5 files changed, 298 insertions(+), 42 deletions(-) diff --git a/fun.py b/fun.py index 883062eba..b2e684472 100644 --- a/fun.py +++ b/fun.py @@ -11,11 +11,17 @@ # INVARIANTS +OFS_DELTA = 6 +REF_DELTA = 7 type_id_to_type_map = { + 0 : "", # EXT 1 1 : "commit", 2 : "tree", 3 : "blob", - 4 : "tag" + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA } # used when dealing with larger streams @@ -42,30 +48,23 @@ def loose_object_header_info(m): type_name, size = hdr[:hdr.find("\0")].split(" ") return type_name, int(size) -def object_header_info(m): - """:return: tuple(type_string, uncompressed_size_in_bytes - :param mmap: mapped memory map. It will be - seeked to the actual start of the object contents, which can be used - to initialize a zlib decompress object. - :note: This routine can only handle new-style objects which are assumably contained - in packs - """ - assert not is_loose_object(m), "Use loose_object_header_info instead" - +def pack_object_header_info(data): + """:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" c = b0 # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size while c & 0x80: - c = ord(m[i]) + c = ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 # END character loop - # finally seek the map to the start of the data stream - m.seek(i) try: return (type_id_to_type_map[type_id], size) except KeyError: diff --git a/pack.py b/pack.py index 2ffc64f52..377963053 100644 --- a/pack.py +++ b/pack.py @@ -1,4 +1,4 @@ -"""Contains PackIndex and PackFile implementations""" +"""Contains PackIndexFile and PackFile implementations""" from util import ( LockedFD, LazyMixin, @@ -6,14 +6,17 @@ unpack_from ) +from fun import ( + pack_object_header_info + ) from struct import ( pack, ) -__all__ = ('PackIndex', 'Pack') +__all__ = ('PackIndexFile', 'PackFile') -class PackIndex(LazyMixin): +class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -26,7 +29,7 @@ class PackIndex(LazyMixin): _sha_list_offset = 8 + 1024 def __init__(self, indexpath): - super(PackIndex, self).__init__() + super(PackIndexFile, self).__init__() self._indexpath = indexpath def _set_cache_(self, attr): @@ -121,9 +124,9 @@ def _initialize(self): self._fanout_table = self._read_fanout((self._version == 2) * 8) if self._version == 2: - self._crc_list_offset = self._sha_list_offset + self.size * 20 - self._pack_offset = self._crc_list_offset + self.size * 4 - self._pack_64_offset = self._pack_offset + self.size * 4 + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base def _read_fanout(self, byte_offset): @@ -139,21 +142,17 @@ def _read_fanout(self, byte_offset): #} END initialization #{ Properties - @property def version(self): return self._version - @property def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - @property def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._data[-40:-20] - @property def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._data[-20:] @@ -186,6 +185,128 @@ def sha_to_index(self, sha): #} END properties -class Pack(LazyMixin): - """A pack is a file written according to the Version 2 for git packs""" +class PackFile(LazyMixin): + """A pack is a file written according to the Version 2 for git packs + As we currently use memory maps, it could be assumed that the maximum size of + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + :note: at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_data', '_size', '_version') + + # offset into our data at which the first object starts + _first_object_offset = 3*4 + 8 + + def __init__(self, packpath): + self._packpath = packpath + + def _set_cache_(self, attr): + if attr == '_data': + ldb = LockedFD(self._packpath) + fd = ldb.open() + self._data = file_contents_ro(fd) + ldb.rollback() + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + else: + # read the header information + type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) + assert type_id == "PACK", "Pack file format is invalid: %r" % type_id + assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version + # END handle header + + def _iter_objects(self, start_offset, as_stream): + """Handle the actual iteration of objects within this pack""" + size = len(self._data) + cur_offset = start_offset or self._first_object_offset + + while cur_offset < size: + type_id, uncomp_size, data_offset = pack_object_header_info(buffer(self._data, cur_offset)) + + # if type_id + # END until we have read everything + + #{ Interface + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._data[-20:] + + #} END interface + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + raise NotImplementedError() + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + raise NotImplementedError() + + #} END Read-Database like Interface + + +class PackFileEntity(object): + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ('_index', '_pack') + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, basename): + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + + def _iter_objects(self, as_stream): + raise NotImplementedError + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + :param sha: 20 byte sha1 + :return: OInfo instance""" + raise NotImplementedError() + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + :param sha: 20 byte sha1 + :return: OStream instance""" + raise NotImplementedError() + + #} END Read-Database like Interface + + #{ Interface + + def info_iter(self): + """:return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """:return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + #} Interface diff --git a/stream.py b/stream.py index 44c7b945a..b30ec1ec9 100644 --- a/stream.py +++ b/stream.py @@ -11,7 +11,11 @@ zlib ) -__all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', +from fun import type_id_to_type_map + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream', 'DecompressMemMapReader', 'FDCompressedSha1Writer') @@ -55,8 +59,42 @@ def type(self): def size(self): return self[2] #} END interface - - + + +class OPackInfo(OInfo): + """As OInfo, but provides a type_id property to retrieve the numerical type id""" + __slots__ = tuple() + + @property + def type(self): + return type_id_to_type_map[self[1]] + + #{ Interface + + @property + def type_id(self): + return self[1] + + #} interface + + +class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the base_offset, being an offset into the pack at which our base + can be found""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, delta_info): + return tuple.__new__(cls, (sha, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + class OStream(OInfo): """Base for object streams retrieved from the database, providing additional information about the stream. @@ -76,6 +114,46 @@ def __init__(self, *args, **kwargs): def read(self, size=-1): return self[3].read(size) + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, delta_info, stream): + return tuple.__new__(cls, (sha, type, size, delta_info, stream)) + + + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] #} END stream reader interface diff --git a/test/test_pack.py b/test/test_pack.py index 199b6c61c..14d8a2496 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -6,23 +6,35 @@ fixture_path ) from gitdb.pack import ( - PackIndex + PackIndexFile, + PackFile ) +from gitdb.util import to_bin_sha import os +#{ Utilities +def bin_sha_from_filename(filename): + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) +#} END utilities + class TestPack(TestBase): - packindexfile_v2 = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') - packindexfile_v1 = fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx') + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + def _assert_index_file(self, index, version, size): - assert index.packfile_checksum != index.indexfile_checksum - assert index.version == version - assert index.size == size + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size # get all data of all objects - for oidx in xrange(index.size): + for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) @@ -34,14 +46,32 @@ def _assert_index_file(self, index, version, size): assert entry[2] == index.crc(oidx) # END for each object index in indexfile + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + def test_pack_index(self): # check version 1 and 2 - index = PackIndex(self.packindexfile_v1) - self._assert_index_file(index, 1, 67) - - index = PackIndex(self.packindexfile_v2) - self._assert_index_file(index, 2, 30) + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + def test_pack_entity(self): + # TODO: + pass + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + pass diff --git a/test/test_stream.py b/test/test_stream.py index 4f022286e..7c9097961 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -39,15 +39,43 @@ def test_streams(self): assert info.type == str_blob_type assert info.size == s + # test pack info + # provides type_id + blob_id = 3 + pinfo = OPackInfo(sha, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + + dpinfo = ODeltaPackInfo(sha, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + + # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream ostream.read(15) stream._assert() assert stream.bytes == 15 ostream.read(20) assert stream.bytes == 20 + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + # derive with own args DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() From 0650892246e99b60448d4a168ea36f84236b97a4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 01:10:46 +0200 Subject: [PATCH 0015/3719] moved all info and stream base classes into new module, base, as well as the respective tests were moved to test_base Adjusted PackStream and PackInfo classes not to contain the sha field anymore streams: DecompressMemMapReader now parses its header on demand if it is not set, using the mose useful 3 lines ever, LazyMixin --- __init__.py | 1 + base.py | 272 +++++++++++++++++++++++++++++++++++++++++++ db/git.py | 2 +- db/loose.py | 9 +- pack.py | 76 +++++++++++- stream.py | 273 ++------------------------------------------ test/db/lib.py | 5 +- test/test_base.py | 90 +++++++++++++++ test/test_stream.py | 78 +------------ 9 files changed, 461 insertions(+), 345 deletions(-) create mode 100644 base.py create mode 100644 test/test_base.py diff --git a/__init__.py b/__init__.py index 8b0e47b19..d79788fc4 100644 --- a/__init__.py +++ b/__init__.py @@ -14,5 +14,6 @@ def _init_externals(): # default imports from db import * +from base import * from stream import * diff --git a/base.py b/base.py new file mode 100644 index 000000000..d12181229 --- /dev/null +++ b/base.py @@ -0,0 +1,272 @@ +"""Module with basic data structures - they are designed to be lightweight and fast""" +from util import ( + to_hex_sha, + to_bin_sha, + zlib + ) + +from fun import type_id_to_type_map + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + +#{ ODB Bases + +class OInfo(tuple): + """Carries information about an object in an ODB, provdiing information + about the sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.sha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def sha(self): + return self[0] + + @property + def type(self): + return self[1] + + @property + def size(self): + return self[2] + #} END interface + + +class OPackInfo(tuple): + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha""" + __slots__ = tuple() + + def __new__(cls, type, size): + return tuple.__new__(cls, (type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def type(self): + return type_id_to_type_map[self[0]] + + @property + def type_id(self): + return self[0] + + @property + def size(self): + return self[1] + + #} END interface + + +class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the base_offset, being an offset into the pack at which our base + can be found""" + __slots__ = tuple() + + def __new__(cls, type, size, delta_info): + return tuple.__new__(cls, (type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[2] + #} END interface + + +class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[2].read(size) + + @property + def stream(self): + return self[2] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, type, size, delta_info, stream): + return tuple.__new__(cls, (type, size, delta_info, stream)) + + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_sha(self, sha): + self[0] = sha + + def _sha(self): + return self[0] + + sha = property(_sha, _set_sha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def sha(self): + return self[0] + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases + diff --git a/db/git.py b/db/git.py index d2477d7b1..0488bc601 100644 --- a/db/git.py +++ b/db/git.py @@ -1,5 +1,5 @@ -from gitdb.stream import ( +from gitdb.base import ( OInfo, OStream ) diff --git a/db/loose.py b/db/loose.py index 37aad8c6f..109782fdc 100644 --- a/db/loose.py +++ b/db/loose.py @@ -13,11 +13,14 @@ from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, - Sha1Writer, - OStream, - OInfo + Sha1Writer ) +from gitdb.base import ( + OStream, + OInfo + ) + from gitdb.util import ( ENOENT, to_hex_sha, diff --git a/pack.py b/pack.py index 377963053..ab394daba 100644 --- a/pack.py +++ b/pack.py @@ -7,8 +7,21 @@ ) from fun import ( - pack_object_header_info + pack_object_header_info, + OFS_DELTA, + REF_DELTA ) + +from base import ( + OPackInfo, + OPackStream, + ODeltaPackInfo, + ODeltaPackStream, + ) +from stream import ( + DecompressMemMapReader, + ) + from struct import ( pack, ) @@ -16,6 +29,60 @@ __all__ = ('PackIndexFile', 'PackFile') + +#{ Utilities + +def pack_object_at(data, as_stream): + """ + :return: info or stream object of the correct type according to the type + of the object, REF_DELTAS will not be resolved in case a stream is desired. + The resulting ODeltaPackStream will have None instead of a stream. + :param data: random accessable data at which the header of an object can be read + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only + :note: a bit redundant, but it needs to be as fast as possible !""" + type_id, uncomp_size, data_offset = pack_object_header_info(data) + + if type_id == OFS_DELTA: + i = 0 + delta_offset = 0 + s = 7 + while c & 0x80: + c = ord(data[i]) + i += 1 + delta_offset += (c & 0x7f) << s + s += 7 + # END character loop + if as_stream: + stream = DecompressMemMapReader(buffer(data, i), False) + return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) + else: + return ODeltaPackInfo(type_id, uncomp_size, delta_offset) + # END handle stream + elif type_id == REF_DELTA: + ref_sha = data[:20] + if as_stream: + stream = DecompressMemMapReader(buffer(data, 20), False) + return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) + else: + return ODeltaPackInfo(type_id, uncomp_size, ref_sha) + # END handle stream + else: + # assume its a base object + if as_stream: + # if no size is given, it will read the header on first access + stream = DecompressMemMapReader(buffer(data, data_offset), False) + return OPackStream(type_id, uncomp_size, stream) + else: + return OPackInfo(type_id, uncomp_size) + # END handle as_stream + # END handle type id + + +#} END utilities + + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -222,13 +289,14 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream): """Handle the actual iteration of objects within this pack""" - size = len(self._data) + data = self._data + size = len(data) cur_offset = start_offset or self._first_object_offset while cur_offset < size: - type_id, uncomp_size, data_offset = pack_object_header_info(buffer(self._data, cur_offset)) + ostream = pack_object_at(buffer(data, cur_offset), True) + # TODO: Decompressor needs to track the size of bytes actually decompressed - # if type_id # END until we have read everything #{ Interface diff --git a/stream.py b/stream.py index b30ec1ec9..f759267a5 100644 --- a/stream.py +++ b/stream.py @@ -3,279 +3,23 @@ import errno from util import ( - to_hex_sha, - to_bin_sha, + LazyMixin, make_sha, write, close, zlib ) -from fun import type_id_to_type_map - -__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream', - 'DecompressMemMapReader', 'FDCompressedSha1Writer') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') # ZLIB configuration # used when compressing objects - 1 to 9 ( slowest ) Z_BEST_SPEED = 1 - -#{ ODB Bases - -class OInfo(tuple): - """Carries information about an object in an ODB, provdiing information - about the sha of the object, the type_string as well as the uncompressed size - in bytes. - - It can be accessed using tuple notation and using attribute access notation:: - - assert dbi[0] == dbi.sha - assert dbi[1] == dbi.type - assert dbi[2] == dbi.size - - The type is designed to be as lighteight as possible.""" - __slots__ = tuple() - - def __new__(cls, sha, type, size): - return tuple.__new__(cls, (sha, type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - @property - def sha(self): - return self[0] - - @property - def type(self): - return self[1] - - @property - def size(self): - return self[2] - #} END interface - - -class OPackInfo(OInfo): - """As OInfo, but provides a type_id property to retrieve the numerical type id""" - __slots__ = tuple() - - @property - def type(self): - return type_id_to_type_map[self[1]] - - #{ Interface - - @property - def type_id(self): - return self[1] - - #} interface - - -class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, - or the base_offset, being an offset into the pack at which our base - can be found""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, delta_info): - return tuple.__new__(cls, (sha, type, size, delta_info)) - - #{ Interface - @property - def delta_info(self): - return self[3] - #} END interface - - -class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional - information about the stream. - Generally, ODB streams are read-only as objects are immutable""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - - def __init__(self, *args, **kwargs): - tuple.__init__(self) - - #{ Stream Reader Interface - - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface - - -class OPackStream(OPackInfo): - """Next to pack object information, a stream outputting an undeltified base object - is provided""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - #{ Stream Reader Interface - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface - - -class ODeltaPackStream(ODeltaPackInfo): - """Provides a stream outputting the uncompressed offset delta information""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, delta_info, stream): - return tuple.__new__(cls, (sha, type, size, delta_info, stream)) - - - #{ Stream Reader Interface - def read(self, size=-1): - return self[4].read(size) - - @property - def stream(self): - return self[4] - #} END stream reader interface - - -class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow - the ODB to record information about the operations outcome right in this instance. - - It provides interfaces for the OStream and a StreamReader to allow the instance - to blend in without prior conversion. - - The only method your content stream must support is 'read'""" - __slots__ = tuple() - - def __new__(cls, type, size, stream, sha=None): - return list.__new__(cls, (sha, type, size, stream, None)) - - def __init__(self, type, size, stream, sha=None): - list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface - - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) - - def _error(self): - """:return: the error that occurred when processing the stream, or None""" - return self[4] - - def _set_error(self, exc): - """Set this input stream to the given exc, may be None to reset the error""" - self[4] = exc - - error = property(_error, _set_error) - - #} END interface - - #{ Stream Reader Interface - - def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on - to our internal stream""" - return self[3].read(size) - - #} END stream reader interface - - #{ interface - - def _set_sha(self, sha): - self[0] = sha - - def _sha(self): - return self[0] - - sha = property(_sha, _set_sha) - - - def _type(self): - return self[1] - - def _set_type(self, type): - self[1] = type - - type = property(_type, _set_type) - - def _size(self): - return self[2] - - def _set_size(self, size): - self[2] = size - - size = property(_size, _set_size) - - def _stream(self): - return self[3] - - def _set_stream(self, stream): - self[3] = stream - - stream = property(_stream, _set_stream) - - #} END odb info interface - - -class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in - the queried database. The exception attribute provides more information about - the cause of the issue""" - __slots__ = tuple() - - def __new__(cls, sha, exc): - return tuple.__new__(cls, (sha, exc)) - - def __init__(self, sha, exc): - tuple.__init__(self, (sha, exc)) - - @property - def sha(self): - return self[0] - - @property - def error(self): - """:return: exception instance explaining the failure""" - return self[1] - - -class InvalidOStream(InvalidOInfo): - """Carries information about an invalid ODB stream""" - __slots__ = tuple() - -#} END ODB Bases - - #{ RO Streams -class DecompressMemMapReader(object): +class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly @@ -296,19 +40,26 @@ class DecompressMemMapReader(object): max_read_size = 512*1024 # currently unused - def __init__(self, m, close_on_deletion, size): + def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no size""" self._m = m self._zip = zlib.decompressobj() self._buf = None # buffer of decompressed bytes self._buflen = 0 # length of bytes in buffer - self._s = size # size of uncompressed data to read in total + if size is not None: + self._s = size # size of uncompressed data to read in total self._br = 0 # num uncompressed bytes read self._cws = 0 # start byte of compression window self._cwe = 0 # end byte of compression window self._close = close_on_deletion # close the memmap on deletion ? + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + def __del__(self): if self._close: self._m.close() diff --git a/test/db/lib.py b/test/db/lib.py index 35823059b..cf752741b 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -6,8 +6,9 @@ TestBase ) -from gitdb.stream import ( - Sha1Writer, +from gitdb.stream import Sha1Writer + +from gitdb.base import ( IStream, OStream, OInfo diff --git a/test/test_base.py b/test/test_base.py new file mode 100644 index 000000000..f7ddebaa2 --- /dev/null +++ b/test/test_base.py @@ -0,0 +1,90 @@ +"""Test for object db""" +from lib import ( + TestBase, + DummyStream, + DeriveTest, + ) + +from gitdb import * +from gitdb.util import ( + NULL_HEX_SHA + ) + +from gitdb.typ import ( + str_blob_type + ) + + +class TestBaseTypes(TestBase): + + def test_streams(self): + # test info + sha = NULL_HEX_SHA + s = 20 + info = OInfo(sha, str_blob_type, s) + assert info.sha == sha + assert info.type == str_blob_type + assert info.size == s + + # test pack info + # provides type_id + blob_id = 3 + pinfo = OPackInfo(blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + + dpinfo = ODeltaPackInfo(blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.sha == None + istream.sha = sha + assert istream.sha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/test/test_stream.py b/test/test_stream.py index 7c9097961..69a93ad0f 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -2,7 +2,6 @@ from lib import ( TestBase, DummyStream, - DeriveTest, Sha1Writer, make_bytes, make_object @@ -18,7 +17,6 @@ str_blob_type ) -from cStringIO import StringIO import tempfile import os @@ -29,78 +27,6 @@ class TestStream(TestBase): """Test stream classes""" data_sizes = (15, 10000, 1000*1024+512) - - def test_streams(self): - # test info - sha = NULL_HEX_SHA - s = 20 - info = OInfo(sha, str_blob_type, s) - assert info.sha == sha - assert info.type == str_blob_type - assert info.size == s - - # test pack info - # provides type_id - blob_id = 3 - pinfo = OPackInfo(sha, blob_id, s) - assert pinfo.type == str_blob_type - assert pinfo.type_id == blob_id - - dpinfo = ODeltaPackInfo(sha, blob_id, s, sha) - assert dpinfo.type == str_blob_type - assert dpinfo.type_id == blob_id - assert dpinfo.delta_info == sha - - - # test ostream - stream = DummyStream() - ostream = OStream(*(info + (stream, ))) - assert ostream.stream is stream - ostream.read(15) - stream._assert() - assert stream.bytes == 15 - ostream.read(20) - assert stream.bytes == 20 - - # test packstream - postream = OPackStream(*(pinfo + (stream, ))) - assert postream.stream is stream - postream.read(10) - stream._assert() - assert stream.bytes == 10 - - # test deltapackstream - dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream - dpostream.read(5) - stream._assert() - assert stream.bytes == 5 - - # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - - # test istream - istream = IStream(str_blob_type, s, stream) - assert istream.sha == None - istream.sha = sha - assert istream.sha == sha - - assert len(istream.binsha) == 20 - assert len(istream.hexsha) == 40 - - assert istream.size == s - istream.size = s * 2 - istream.size == s * 2 - assert istream.type == str_blob_type - istream.type = "something" - assert istream.type == "something" - assert istream.stream is stream - istream.stream = None - assert istream.stream is None - - assert istream.error is None - istream.error = Exception() - assert isinstance(istream.error, Exception) def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): """Make stream tests - the orig_stream is seekable, allowing it to be @@ -145,6 +71,10 @@ def test_decompress_reader(self): type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) assert type == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) else: # here we need content data zdata = zlib.compress(cdata) From bf4437ef45d9115aa2716e1c722c3938b6976803 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 13:37:59 +0200 Subject: [PATCH 0016/3719] DecompressMemMapReader: implemented compressed bytes counting, including test. This is required to properly read packs without the use of an index --- ext/async | 2 +- pack.py | 6 +-- stream.py | 105 +++++++++++++++++++++++++++++--------------- test/test_stream.py | 19 ++++---- 4 files changed, 82 insertions(+), 50 deletions(-) diff --git a/ext/async b/ext/async index af0040b0f..796b5e94f 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit af0040b0f3c6ede3be5b2d6bc69f6ea5ac53c36c +Subproject commit 796b5e94f19dfc36a3fb251468192373c76510b0 diff --git a/pack.py b/pack.py index ab394daba..2c01f0452 100644 --- a/pack.py +++ b/pack.py @@ -54,7 +54,7 @@ def pack_object_at(data, as_stream): s += 7 # END character loop if as_stream: - stream = DecompressMemMapReader(buffer(data, i), False) + stream = DecompressMemMapReader(buffer(data, i), False, uncomp_size) return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) else: return ODeltaPackInfo(type_id, uncomp_size, delta_offset) @@ -62,7 +62,7 @@ def pack_object_at(data, as_stream): elif type_id == REF_DELTA: ref_sha = data[:20] if as_stream: - stream = DecompressMemMapReader(buffer(data, 20), False) + stream = DecompressMemMapReader(buffer(data, 20), False, uncomp_size) return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) else: return ODeltaPackInfo(type_id, uncomp_size, ref_sha) @@ -267,7 +267,7 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 + 8 + _first_object_offset = 3*4 def __init__(self, packpath): self._packpath = packpath diff --git a/stream.py b/stream.py index f759267a5..de7ddcd86 100644 --- a/stream.py +++ b/stream.py @@ -1,6 +1,8 @@ from cStringIO import StringIO import errno +import mmap +import os from util import ( LazyMixin, @@ -13,10 +15,6 @@ __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') -# ZLIB configuration -# used when compressing objects - 1 to 9 ( slowest ) -Z_BEST_SPEED = 1 - #{ RO Streams class DecompressMemMapReader(LazyMixin): @@ -36,7 +34,8 @@ class DecompressMemMapReader(LazyMixin): times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close') + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') max_read_size = 512*1024 # currently unused @@ -52,6 +51,8 @@ def __init__(self, m, close_on_deletion, size=None): self._br = 0 # num uncompressed bytes read self._cws = 0 # start byte of compression window self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info self._close = close_on_deletion # close the memmap on deletion ? def _set_cache_(self, attr): @@ -85,6 +86,8 @@ def _parse_header_info(self): self._buf = StringIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend + self._phi = True + return type, size @classmethod @@ -98,7 +101,55 @@ def new(self, m, close_on_deletion=False): inst = DecompressMemMapReader(m, close_on_deletion, 0) type, size = inst._parse_header_info() return type, size, inst + + def compressed_bytes_read(self): + """:return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we can only hope it works. + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and hasattr(self._zip, 'status') and self._zip.status == zlib.Z_OK: + # manipulate the bytes-read to allow our own read method to coninute + # but keep the window at its current position + self._br = 0 + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + return self._cbr - len(self._zip.unused_data) + def seek(self, offset, whence=os.SEEK_SET): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + def read(self, size=-1): if size < 1: size = self._s - self._br @@ -109,33 +160,8 @@ def read(self, size=-1): if size == 0: return str() # END handle depletion - - # protect from memory peaks - # If he tries to read large chunks, our memory patterns get really bad - # as we end up copying a possibly huge chunk from our memory map right into - # memory. This might not even be possible. Nonetheless, try to dampen the - # effect a bit by reading in chunks, returning a huge string in the end. - # Our performance now depends on StringIO. This way we don't need two large - # buffers in peak times, but only one large one in the end which is - # the return buffer - # NO: We don't do it - if the user thinks its best, he is right. If he - # has trouble, he will start reading in chunks. According to our tests - # its still faster if we read 10 Mb at once instead of chunking it. - - # if size > self.max_read_size: - # sio = StringIO() - # while size: - # read_size = min(self.max_read_size, size) - # data = self.read(read_size) - # sio.write(data) - # size -= len(data) - # if len(data) < read_size: - # break - # # END data loop - # sio.seek(0) - # return sio.getvalue() - # # END handle maxread - # + + # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream @@ -186,8 +212,7 @@ def read(self, size=-1): # if window is too small, make it larger so zip can decompress something - win_size = self._cwe - self._cws - if win_size < 8: + if self._cwe - self._cws < 8: self._cwe = self._cws + 8 # END adjust winsize @@ -196,10 +221,18 @@ def read(self, size=-1): # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) - + dcompdat = self._zip.decompress(indata, size) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) + + print size, self._br, self._cbr, len(indata), self._cws, self._cwe, len(self._zip.unused_data), len(self._zip.unconsumed_tail) + if dat: dcompdat = dat + dcompdat @@ -252,7 +285,7 @@ class FDCompressedSha1Writer(Sha1Writer): def __init__(self, fd): super(FDCompressedSha1Writer, self).__init__() self.fd = fd - self.zip = zlib.compressobj(Z_BEST_SPEED) + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) #{ Stream Interface diff --git a/test/test_stream.py b/test/test_stream.py index 69a93ad0f..41f2b235a 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -49,12 +49,20 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert rest == cdata[-len(rest):] # END handle rest + if isinstance(stream, DecompressMemMapReader): + assert len(stream._m) == stream.compressed_bytes_read() + # END handle special type + rewind_stream(stream) # read everything rdata = stream.read() assert rdata == cdata + if isinstance(stream, DecompressMemMapReader): + assert len(stream._m) == stream.compressed_bytes_read() + # END handle special type + def test_decompress_reader(self): for close_on_deletion in range(2): for with_size in range(2): @@ -82,15 +90,7 @@ def test_decompress_reader(self): assert reader._s == len(cdata) # END get reader - def rewind(r): - r._zip = zlib.decompressobj() - r._br = r._cws = r._cwe = 0 - if with_size: - r._parse_header_info() - # END skip header - # END make rewind func - - self._assert_stream_reader(reader, cdata, rewind) + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) # put in a dummy stream for closing dummy = DummyStream() @@ -99,7 +99,6 @@ def rewind(r): assert not dummy.closed del(reader) assert dummy.closed == close_on_deletion - #zdi# # END for each datasize # END whether size should be used # END whether stream should be closed when deleted From b6db082874b528b431d90de898a3061b2b6d9c36 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 14:01:45 +0200 Subject: [PATCH 0017/3719] DecompressMemMapReader: improved compressed_bytes_read method with alternate route which is a bit less efficient, but works without a custom zlib --- stream.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/stream.py b/stream.py index de7ddcd86..fb58125bf 100644 --- a/stream.py +++ b/stream.py @@ -120,16 +120,26 @@ def compressed_bytes_read(self): # until it is finished, to yield the actual number of compressed bytes # belonging to the decompressed object # We are using a custom zlib module for this, if its not present, - # we can only hope it works. + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + # Only scrub the stream forward if we are officially done with the # bytes we were to have. - if self._br == self._s and hasattr(self._zip, 'status') and self._zip.status == zlib.Z_OK: + if self._br == self._s and not self._zip.unused_data: # manipulate the bytes-read to allow our own read method to coninute # but keep the window at its current position self._br = 0 - while self._zip.status == zlib.Z_OK: - self.read(mmap.PAGESIZE) - # END scrub-loop + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + # reset bytes read, just to be sure self._br = self._s # END handle stream scrubbing @@ -231,8 +241,6 @@ def read(self, size=-1): self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) - print size, self._br, self._cbr, len(indata), self._cws, self._cwe, len(self._zip.unused_data), len(self._zip.unconsumed_tail) - if dat: dcompdat = dat + dcompdat From 4977bc52938c058123f2a3f6e0dbf6fc404550dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 17:36:57 +0200 Subject: [PATCH 0018/3719] implemented direct pack reading - currently not all information is passed on, the absolute offset into the packfile could be interesting to the caller --- fun.py | 4 +-- pack.py | 82 ++++++++++++++++++++++++++++++++++------------- stream.py | 29 ++++++++++++++++- test/test_pack.py | 3 ++ 4 files changed, 92 insertions(+), 26 deletions(-) diff --git a/fun.py b/fun.py index b2e684472..bf223e904 100644 --- a/fun.py +++ b/fun.py @@ -53,7 +53,7 @@ def pack_object_header_info(data): The type_id should be interpreted according to the ``type_id_to_type_map`` map The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" - c = b0 # first byte + c = ord(data[0]) # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size @@ -66,7 +66,7 @@ def pack_object_header_info(data): # END character loop try: - return (type_id_to_type_map[type_id], size) + return (type_id, size, i) except KeyError: # invalid object type - we could try to be smart now and decode part # of the stream to get the info, problem is that we had trouble finding diff --git a/pack.py b/pack.py index 2c01f0452..fa5e43b31 100644 --- a/pack.py +++ b/pack.py @@ -8,6 +8,8 @@ from fun import ( pack_object_header_info, + stream_copy, + chunk_size, OFS_DELTA, REF_DELTA ) @@ -20,6 +22,7 @@ ) from stream import ( DecompressMemMapReader, + NullStream ) from struct import ( @@ -34,50 +37,61 @@ def pack_object_at(data, as_stream): """ - :return: info or stream object of the correct type according to the type - of the object, REF_DELTAS will not be resolved in case a stream is desired. - The resulting ODeltaPackStream will have None instead of a stream. + :return: tuple(num_header_bytes, PackInfo|PackStream) + Tuple of number of additional bytes read from data until the data stream begins + and object of the correct type according to the type of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. :param data: random accessable data at which the header of an object can be read :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only :note: a bit redundant, but it needs to be as fast as possible !""" type_id, uncomp_size, data_offset = pack_object_header_info(data) - + total_offset = None # set later, actual offset until data stream begins + obj = None if type_id == OFS_DELTA: - i = 0 + i = data_offset delta_offset = 0 s = 7 - while c & 0x80: + while True: c = ord(data[i]) - i += 1 delta_offset += (c & 0x7f) << s + i += 1 + if not (c & 0x80): + break s += 7 # END character loop + total_offset = i if as_stream: - stream = DecompressMemMapReader(buffer(data, i), False, uncomp_size) - return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) + stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) + obj = ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) else: - return ODeltaPackInfo(type_id, uncomp_size, delta_offset) + obj = ODeltaPackInfo(type_id, uncomp_size, delta_offset) # END handle stream elif type_id == REF_DELTA: - ref_sha = data[:20] + total_offset = data_offset+20 + ref_sha = data[data_offset:total_offset] + if as_stream: - stream = DecompressMemMapReader(buffer(data, 20), False, uncomp_size) - return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) + stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) + obj = ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) else: - return ODeltaPackInfo(type_id, uncomp_size, ref_sha) + obj = ODeltaPackInfo(type_id, uncomp_size, ref_sha) # END handle stream else: + total_offset = data_offset # assume its a base object if as_stream: # if no size is given, it will read the header on first access - stream = DecompressMemMapReader(buffer(data, data_offset), False) - return OPackStream(type_id, uncomp_size, stream) + stream = DecompressMemMapReader(buffer(data, data_offset), False, uncomp_size) + obj = OPackStream(type_id, uncomp_size, stream) else: - return OPackInfo(type_id, uncomp_size) + obj = OPackInfo(type_id, uncomp_size) # END handle as_stream # END handle type id + return total_offset, obj + #} END utilities @@ -267,7 +281,8 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 + _first_object_offset = 3*4 # header bytes + _footer_size = 20 # final sha def __init__(self, packpath): self._packpath = packpath @@ -287,16 +302,28 @@ def _set_cache_(self, attr): assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version # END handle header - def _iter_objects(self, start_offset, as_stream): + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data - size = len(data) + content_size = len(data) - self._footer_size cur_offset = start_offset or self._first_object_offset - while cur_offset < size: - ostream = pack_object_at(buffer(data, cur_offset), True) - # TODO: Decompressor needs to track the size of bytes actually decompressed + null = NullStream() + while cur_offset < content_size: + header_offset, ostream = pack_object_at(buffer(data, cur_offset), True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + cur_offset += header_offset + ostream.stream.compressed_bytes_read() + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream # END until we have read everything #{ Interface @@ -329,6 +356,15 @@ def stream(self, offset): :return: OPackStream instance, the actual type differs depending on the type_id attribute""" raise NotImplementedError() + def stream_iter(self, start_offset=0): + """:return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + :note: Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + #} END Read-Database like Interface diff --git a/stream.py b/stream.py index fb58125bf..898059def 100644 --- a/stream.py +++ b/stream.py @@ -17,6 +17,21 @@ #{ RO Streams +class NullStream(object): + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + + class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand @@ -144,7 +159,9 @@ def compressed_bytes_read(self): self._br = self._s # END handle stream scrubbing - return self._cbr - len(self._zip.unused_data) + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading @@ -243,7 +260,17 @@ def read(self, size=-1): if dat: dcompdat = dat + dcompdat + # END prepend our cached data + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. + # Recursively resolve that. + # Note: dcompdat can be empty even though we still appear to have bytes + # to read, if we are called by compressed_bytes_read - it manipulates + # us to empty the stream + if dcompdat and len(dcompdat) < size and self._br < self._s: + dcompdat += self.read(size-len(dcompdat)) + # END handle special case return dcompdat #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index 14d8a2496..a5234a6ce 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -52,6 +52,9 @@ def _assert_pack_file(self, pack, version, size): assert pack.size() == size assert len(pack.checksum()) == 20 + objs = list(pack.stream_iter()) + assert len(objs) == size + def test_pack_index(self): # check version 1 and 2 From ca8236451439490670822103b475df9925884e32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 21:57:22 +0200 Subject: [PATCH 0019/3719] Implemented offset based pack object collection including test, next up is the actual stream delta handling --- base.py | 59 +++++++++++------ fun.py | 9 +++ pack.py | 157 +++++++++++++++++++++++++++++++--------------- test/test_base.py | 12 +++- test/test_pack.py | 29 ++++++++- 5 files changed, 191 insertions(+), 75 deletions(-) diff --git a/base.py b/base.py index d12181229..aa917858d 100644 --- a/base.py +++ b/base.py @@ -5,7 +5,10 @@ zlib ) -from fun import type_id_to_type_map +from fun import ( + type_id_to_type_map, + type_to_type_id_map + ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', @@ -41,6 +44,10 @@ def sha(self): @property def type(self): return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] @property def size(self): @@ -50,28 +57,40 @@ def size(self): class OPackInfo(tuple): """As OInfo, but provides a type_id property to retrieve the numerical type id, and - does not include a sha""" + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the abosolute + location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - def __new__(cls, type, size): - return tuple.__new__(cls, (type, size)) + def __new__(cls, packoffset, dataoffset, type, size): + return tuple.__new__(cls, (packoffset, dataoffset, type, size)) def __init__(self, *args): tuple.__init__(self) #{ Interface + @property + def pack_offset(self): + return self[0] + + @property + def data_offset(self): + return self[1] + @property def type(self): - return type_id_to_type_map[self[0]] + return type_id_to_type_map[self[2]] @property def type_id(self): - return self[0] + return self[2] @property def size(self): - return self[1] + return self[3] #} END interface @@ -79,17 +98,17 @@ def size(self): class ODeltaPackInfo(OPackInfo): """Adds delta specific information, Either the 20 byte sha which points to some object in the database, - or the base_offset, being an offset into the pack at which our base - can be found""" + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" __slots__ = tuple() - def __new__(cls, type, size, delta_info): - return tuple.__new__(cls, (type, size, delta_info)) + def __new__(cls, packoffset, dataoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info)) #{ Interface @property def delta_info(self): - return self[2] + return self[4] #} END interface @@ -123,17 +142,17 @@ class OPackStream(OPackInfo): is provided""" __slots__ = tuple() - def __new__(cls, type, size, stream, *args): + def __new__(cls, packoffset, dataoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (type, size, stream)) + return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[2].read(size) + return self[4].read(size) @property def stream(self): - return self[2] + return self[4] #} END stream reader interface @@ -141,17 +160,17 @@ class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - def __new__(cls, type, size, delta_info, stream): - return tuple.__new__(cls, (type, size, delta_info, stream)) + def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[3].read(size) + return self[5].read(size) @property def stream(self): - return self[3] + return self[5] #} END stream reader interface diff --git a/fun.py b/fun.py index bf223e904..7999c0a92 100644 --- a/fun.py +++ b/fun.py @@ -24,6 +24,15 @@ REF_DELTA : "REF_DELTA" # REFERENCE DELTA } +type_to_type_id_map = dict( + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA + ) + # used when dealing with larger streams chunk_size = 1000*1000 diff --git a/pack.py b/pack.py index fa5e43b31..8de2a47db 100644 --- a/pack.py +++ b/pack.py @@ -1,4 +1,7 @@ """Contains PackIndexFile and PackFile implementations""" +from gitdb.exc import ( + BadObject, + ) from util import ( LockedFD, LazyMixin, @@ -31,67 +34,68 @@ __all__ = ('PackIndexFile', 'PackFile') +_delta_types = (OFS_DELTA, REF_DELTA) #{ Utilities -def pack_object_at(data, as_stream): +def pack_object_at(data, offset, as_stream): """ - :return: tuple(num_header_bytes, PackInfo|PackStream) - Tuple of number of additional bytes read from data until the data stream begins - and object of the correct type according to the type of the object. + :return: PackInfo|PackStream + an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. - :param data: random accessable data at which the header of an object can be read + :param data: random accessable data containing all required information + :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read - the data, otherwise you receive an info object only - :note: a bit redundant, but it needs to be as fast as possible !""" - type_id, uncomp_size, data_offset = pack_object_header_info(data) - total_offset = None # set later, actual offset until data stream begins - obj = None + the data, otherwise you receive an info object only""" + ldata = len(data) # debug + data = buffer(data, offset) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA if type_id == OFS_DELTA: - i = data_offset - delta_offset = 0 - s = 7 - while True: + i = data_rela_offset + c = ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: c = ord(data[i]) - delta_offset += (c & 0x7f) << s i += 1 - if not (c & 0x80): - break - s += 7 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) # END character loop - total_offset = i - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) - obj = ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) - else: - obj = ODeltaPackInfo(type_id, uncomp_size, delta_offset) - # END handle stream + delta_info = delta_offset + total_rela_offset = i + # REF DELTA elif type_id == REF_DELTA: - total_offset = data_offset+20 - ref_sha = data[data_offset:total_offset] - - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) - obj = ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) - else: - obj = ODeltaPackInfo(type_id, uncomp_size, ref_sha) - # END handle stream + total_rela_offset = data_rela_offset+20 + ref_sha = data[data_rela_offset:total_rela_offset] + delta_info = ref_sha + # BASE OBJECT else: - total_offset = data_offset # assume its a base object - if as_stream: - # if no size is given, it will read the header on first access - stream = DecompressMemMapReader(buffer(data, data_offset), False, uncomp_size) - obj = OPackStream(type_id, uncomp_size, stream) - else: - obj = OPackInfo(type_id, uncomp_size) - # END handle as_stream + total_rela_offset = data_rela_offset # END handle type id - return total_offset, obj - + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + if delta_info is None: + return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream) + else: + return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return OPackInfo(offset, abs_data_offset, type_id, uncomp_size) + else: + return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream + + #} END utilities @@ -310,12 +314,12 @@ def _iter_objects(self, start_offset, as_stream=True): null = NullStream() while cur_offset < content_size: - header_offset, ostream = pack_object_at(buffer(data, cur_offset), True) + ostream = pack_object_at(data, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += header_offset + ostream.stream.compressed_bytes_read() + cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand @@ -326,7 +330,7 @@ def _iter_objects(self, start_offset, as_stream=True): yield ostream # END until we have read everything - #{ Interface + #{ Pack Information def size(self): """:return: The amount of objects stored in this pack""" @@ -340,7 +344,58 @@ def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._data[-20:] - #} END interface + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + :param offset: specifies the first byte of the object within this pack""" + out = list() + while True: + ostream = pack_object_at(self._data, offset, True) + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out + + def to_delta_stream(self, stream_list): + """Convert the given list of streams into a stream which resolves deltas + (if availble) when reading from it. + :param stream_list: one or more stream objects. If the first stream is a Delta, + there must be at least two streams in the list. The list's last stream + must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled due to a missing base object""" + if len(stream_list) == 1: + if stream_list[0].type_id in _delta_types: + raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type) + # its an object, no need to resolve anything + return stream_list[0] + # END single object special handling + + if stream_list[-1].type_id in _delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + # just create the respective stream wrapper + raise NotImplementedError() + + + #} END pack specific #{ Read-Database like Interface @@ -348,13 +403,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - raise NotImplementedError() + return pack_object_at(self._data, offset or self._first_object_offset, False) def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - raise NotImplementedError() + return pack_object_at(self._data, offset or self._first_object_offset, True) def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing @@ -390,12 +445,14 @@ def _iter_objects(self, as_stream): def info(self, sha): """Retrieve information about the object identified by the given sha :param sha: 20 byte sha1 + :raise BadObject: :return: OInfo instance""" raise NotImplementedError() def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 + :raise BadObject: :return: OStream instance""" raise NotImplementedError() diff --git a/test/test_base.py b/test/test_base.py index f7ddebaa2..524bf3054 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -21,22 +21,28 @@ def test_streams(self): # test info sha = NULL_HEX_SHA s = 20 + blob_id = 3 + info = OInfo(sha, str_blob_type, s) assert info.sha == sha assert info.type == str_blob_type + assert info.type_id == blob_id assert info.size == s # test pack info # provides type_id - blob_id = 3 - pinfo = OPackInfo(blob_id, s) + pinfo = OPackInfo(0, 1, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + assert pinfo.data_offset == 1 - dpinfo = ODeltaPackInfo(blob_id, s, sha) + dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + assert dpinfo.data_offset == 1 # test ostream diff --git a/test/test_pack.py b/test/test_pack.py index a5234a6ce..f2b0f9481 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -52,8 +52,33 @@ def _assert_pack_file(self, pack, version, size): assert pack.size() == size assert len(pack.checksum()) == 20 - objs = list(pack.stream_iter()) - assert len(objs) == size + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.data_offset == stream.data_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = pack.to_delta_stream(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we havent resolved ( as we are without an index ) + continue + # END get deltastream + # END for each object + assert num_obj == size def test_pack_index(self): From 84c4e5a33aac3010d197a3a92adbff52a83969da Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 00:39:17 +0200 Subject: [PATCH 0020/3719] initial research on possible delta-apply algorithms. True streaming appears only possible if delta opcodes are acessing only sequential memory, but through mmaps, it should still be possible to obtain decent performance even on big files --- pack.py | 6 ++-- stream.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ test/test_pack.py | 8 +++++ 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/pack.py b/pack.py index 8de2a47db..314ae07c1 100644 --- a/pack.py +++ b/pack.py @@ -25,7 +25,8 @@ ) from stream import ( DecompressMemMapReader, - NullStream + DeltaApplyReader, + NullStream, ) from struct import ( @@ -49,7 +50,6 @@ def pack_object_at(data, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - ldata = len(data) # debug data = buffer(data, offset) type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -392,7 +392,7 @@ def to_delta_stream(self, stream_list): # END check stream # just create the respective stream wrapper - raise NotImplementedError() + return DeltaApplyReader(stream_list) #} END pack specific diff --git a/stream.py b/stream.py index 898059def..735924845 100644 --- a/stream.py +++ b/stream.py @@ -272,7 +272,82 @@ def read(self, size=-1): dcompdat += self.read(size-len(dcompdat)) # END handle special case return dcompdat + + +class DeltaApplyReader(LazyMixin): + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_streams", # tuple of our stream objects + "_readers", # list of read methods from our streams + "_mm_target", # memory map of the delta-applied data + ) + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._streams = tuple(stream_list) + self._readers = None # TODO + + def _set_cache_(self, attr): + """If we are here, we apply the actual deltas""" + # fill in delta info structures, providing the source and target buffer + # sizes. + # Allocate private memory map big enough to hold the first base buffer + # It can be swapped out if it is too large. We need random access to it + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + + # NOTE: on py pre 2.5, all memory maps must actually be some kind + # of memory buffer,like StringIO ( ouch ;) ) + + + + # TODO: Once that works, figure out the ordering of the opcodes. If they + # are always in-order/sequential, an alternate implementation could + # use stream access only. Of course this would mean we would read + # all deltas in advance, analyse the opcode ranges to determine a final + # concatenated opcode list which indicates what to copy from which delta + # to which position. This preprocessing would allow true streaming + + def read(self, size=0): + # pass the call to our lazy-loaded delta-applied data + return self._mm_target.read(size) + #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index f2b0f9481..d972bef14 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -77,6 +77,14 @@ def _assert_pack_file(self, pack, version, size): # which we havent resolved ( as we are without an index ) continue # END get deltastream + + # TODO: TestStream._assert_stream_reader does that already, should + # be used instead + # read all + dstream.read() + + # read chunks + # END for each object assert num_obj == size From 6a4eee20486eca91d06d7ba1420c8a31bcf0f4a8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 11:20:06 +0200 Subject: [PATCH 0021/3719] initial version of delta-apply, but more pedandic testing is required --- fun.py | 89 ++++++++++++++++++++++++++++++++++++++++- stream.py | 100 +++++++++++++++++++++++++++++++++++++++------- test/test_pack.py | 6 +-- util.py | 13 ++++++ 4 files changed, 188 insertions(+), 20 deletions(-) diff --git a/fun.py b/fun.py index 7999c0a92..52688ba7b 100644 --- a/fun.py +++ b/fun.py @@ -9,6 +9,7 @@ from util import zlib decompressobj = zlib.decompressobj +import mmap # INVARIANTS OFS_DELTA = 6 @@ -34,7 +35,7 @@ ) # used when dealing with larger streams -chunk_size = 1000*1000 +chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', 'write_object' ) @@ -83,6 +84,26 @@ def pack_object_header_info(data): raise BadObjectType(type_id) # END handle exceptions +def msb_size(data, offset=0): + """:return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i+offset, size + def write_object(type, size, read, write, chunk_size=chunk_size): """Write the object as identified by type, size and source_stream into the target_stream @@ -111,8 +132,15 @@ def stream_copy(read, write, size, chunk_size): # WRITE ALL DATA UP TO SIZE while True: cs = min(chunk_size, size-dbw) - data_len = write(read(cs)) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) dbw += data_len + write(data) if data_len < cs or dbw == size: break # END check for stream end @@ -120,5 +148,62 @@ def stream_copy(read, write, size, chunk_size): return dbw +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): + """Apply data from a delta buffer using a source buffer to the target file, + which will be written to + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf: random access delta data + :param target_file: file like object to write the result to + :note: transcribed to python from the similar routine in patch-delta.c""" + i = 0 + twrite = target_file.write + db = delta_buf + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += i + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + # maybe skip this check ? + if (cp_off + cp_size < cp_size or + cp_off + cp_size > src_buf_size): + break + twrite(src_buf[cp_off:cp_off+cp_size]) + elif c: + twrite(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + #} END routines diff --git a/stream.py b/stream.py index 735924845..5ced7ada0 100644 --- a/stream.py +++ b/stream.py @@ -4,7 +4,14 @@ import mmap import os +from fun import ( + msb_size, + stream_copy, + apply_delta_data + ) + from util import ( + allocate_memory, LazyMixin, make_sha, write, @@ -300,9 +307,11 @@ class DeltaApplyReader(LazyMixin): * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( - "_streams", # tuple of our stream objects - "_readers", # list of read methods from our streams + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read ) def __init__(self, stream_list): @@ -311,31 +320,81 @@ def __init__(self, stream_list): base object onto which to apply the deltas""" assert len(stream_list) > 1, "Need at least one delta and one base stream" - self._streams = tuple(stream_list) - self._readers = None # TODO + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" # fill in delta info structures, providing the source and target buffer # sizes. + buffer_offset_list = list() + final_target_size = None + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + if final_target_size is None: + final_target_size = target_size + # END set final target size + buffer_offset_list.append((buffer(buf, offset), offset)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + # Allocate private memory map big enough to hold the first base buffer - # It can be swapped out if it is too large. We need random access to it + # We need random access to it + bbuf = allocate_memory(base_size) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final # target buffer is smaller than our allocated space, we just use parts - # of it + # of it upon return. + tbuf = allocate_memory(target_size) # for each delta to apply, memory map the decompressed delta and # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. - - # NOTE: on py pre 2.5, all memory maps must actually be some kind - # of memory buffer,like StringIO ( ouch ;) ) - - + for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + + ################################################################ + apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf) + ################################################################ + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size # TODO: Once that works, figure out the ordering of the opcodes. If they # are always in-order/sequential, an alternate implementation could @@ -344,10 +403,21 @@ def _set_cache_(self, attr): # concatenated opcode list which indicates what to copy from which delta # to which position. This preprocessing would allow true streaming - def read(self, size=0): - # pass the call to our lazy-loaded delta-applied data - return self._mm_target.read(size) - + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=os.SEEK_SET): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + self._size #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index d972bef14..5786fbf72 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -78,12 +78,12 @@ def _assert_pack_file(self, pack, version, size): continue # END get deltastream - # TODO: TestStream._assert_stream_reader does that already, should - # be used instead # read all - dstream.read() + assert len(dstream.read()) # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map # END for each object assert num_obj == size diff --git a/util.py b/util.py index 5c2bb540b..f10b71b12 100644 --- a/util.py +++ b/util.py @@ -94,6 +94,19 @@ def stream_copy(source, destination, chunk_size=512*1024): # END reading output stream return br +def allocate_memory(size): + """:return: a file-protocol accessible memory block of the given size""" + try: + return mmap.mmap(-1, size) # read-write by default + except EnvironmentError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return cStringIO.StringIO("\0"*size) + # END handle memory allocation + + def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd :param fd: file descriptor opened for reading From f4b6e272963fefdb1e372b87a09e7c74680d6b52 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 13:24:41 +0200 Subject: [PATCH 0022/3719] Implemented main PackEntity object retrieval method and moved constructor for delta_streams out of the PackFile, into the stream itself where it belongs. All this is still to be tested --- fun.py | 2 + pack.py | 145 ++++++++++++++++++++++++++++++++++------------ stream.py | 49 +++++++++++++++- test/test_pack.py | 9 ++- 4 files changed, 165 insertions(+), 40 deletions(-) diff --git a/fun.py b/fun.py index 52688ba7b..cbc37b2f0 100644 --- a/fun.py +++ b/fun.py @@ -14,6 +14,8 @@ # INVARIANTS OFS_DELTA = 6 REF_DELTA = 7 +delta_types = (OFS_DELTA, REF_DELTA) + type_id_to_type_map = { 0 : "", # EXT 1 1 : "commit", diff --git a/pack.py b/pack.py index 314ae07c1..811acaf98 100644 --- a/pack.py +++ b/pack.py @@ -5,19 +5,24 @@ from util import ( LockedFD, LazyMixin, - file_contents_ro, - unpack_from + unpack_from, + file_contents_ro, ) from fun import ( pack_object_header_info, + type_id_to_type_map, stream_copy, chunk_size, + delta_types, OFS_DELTA, - REF_DELTA + REF_DELTA, + msb_size ) -from base import ( +from base import ( # Amazing ! + OInfo, + OStream, OPackInfo, OPackStream, ODeltaPackInfo, @@ -35,7 +40,7 @@ __all__ = ('PackIndexFile', 'PackFile') -_delta_types = (OFS_DELTA, REF_DELTA) + #{ Utilities @@ -95,8 +100,6 @@ def pack_object_at(data, offset, as_stream): # END handle info # END handle stream - - #} END utilities @@ -355,6 +358,7 @@ def collect_streams(self, offset): the last one is either a full object, or a REF_Delta stream. The latter type needs its reference object to be locked up in an ODB to form a valid delta chain. + If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() while True: @@ -370,31 +374,7 @@ def collect_streams(self, offset): # END handle type # END while chaining streams return out - - def to_delta_stream(self, stream_list): - """Convert the given list of streams into a stream which resolves deltas - (if availble) when reading from it. - :param stream_list: one or more stream objects. If the first stream is a Delta, - there must be at least two streams in the list. The list's last stream - must be a non-delta stream. - :return: Non-Delta OPackStream object whose stream can be used to obtain - the decompressed resolved data - :raise ValueError: if the stream list cannot be handled due to a missing base object""" - if len(stream_list) == 1: - if stream_list[0].type_id in _delta_types: - raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type) - # its an object, no need to resolve anything - return stream_list[0] - # END single object special handling - - if stream_list[-1].type_id in _delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) - # END check stream - - # just create the respective stream wrapper - return DeltaApplyReader(stream_list) - - + #} END pack specific #{ Read-Database like Interface @@ -437,8 +417,58 @@ def __init__(self, basename): self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + def _iter_objects(self, as_stream): - raise NotImplementedError + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + raise NotImplementedError() + + def _object(self, sha, as_stream): + """:return: OInfo or OStream object providing information about the given sha""" + # its a little bit redundant here, but it needs to be efficient + offset = self._index.offset(self._sha_to_index(sha)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + streams[0].seek(0) # assure it can be read by the delta reader + dstream = DeltaApplyReader.new(streams) + + return OStream(sha, dstream.type, target_size, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + + return OInfo(sha, streams[-1].type, target_size) + # END handle stream #{ Read-Database like Interface @@ -447,14 +477,14 @@ def info(self, sha): :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance""" - raise NotImplementedError() + return self._object(sha, as_stream=False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance""" - raise NotImplementedError() + return self._object(sha, as_stream=True) #} END Read-Database like Interface @@ -470,4 +500,47 @@ def stream_iter(self): OStream instances""" return self._iter_objects(as_stream=True) - #} Interface + def collect_streams_at_offset(self, offset): + """As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + + + #} END interface diff --git a/stream.py b/stream.py index 5ced7ada0..10a8e057c 100644 --- a/stream.py +++ b/stream.py @@ -7,7 +7,8 @@ from fun import ( msb_size, stream_copy, - apply_delta_data + apply_delta_data, + delta_types ) from util import ( @@ -19,7 +20,7 @@ zlib ) -__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') #{ RO Streams @@ -418,6 +419,50 @@ def seek(self, offset, whence=os.SEEK_SET): raise ValueError("Can only seek to position 0") # END handle offset self._size + + #{ Interface + + @classmethod + def new(cls, stream_list): + """Convert the given list of streams into a stream which resolves deltas + when reading from it. + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + return cls(stream_list) + + #} END interface + + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index 5786fbf72..1f860961e 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,6 +5,10 @@ with_packs_rw, fixture_path ) +from gitdb.stream import ( + DeltaApplyReader + ) + from gitdb.pack import ( PackIndexFile, PackFile @@ -71,15 +75,16 @@ def _assert_pack_file(self, pack, version, size): # read the stream try: - dstream = pack.to_delta_stream(streams) + dstream = DeltaApplyReader.new(streams) except ValueError: # ignore these, old git versions use only ref deltas, # which we havent resolved ( as we are without an index ) + # Also ignore non-delta streams continue # END get deltastream # read all - assert len(dstream.read()) + assert len(dstream.read()) # read chunks # NOTE: the current implementation is safe, it basically transfers From 001f030cc8c87407d30fe87fe24929cc1edb8f48 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 16:51:39 +0200 Subject: [PATCH 0023/3719] Initial implementation of stream validation - this is the final hurdle, if that works ( which it doesn't for yet for everything ), than the pack reading would officially work --- fun.py | 7 ++- pack.py | 124 ++++++++++++++++++++++++++++++++++++++-------- stream.py | 2 +- test/test_pack.py | 38 ++++++++++++-- util.py | 15 ------ 5 files changed, 146 insertions(+), 40 deletions(-) diff --git a/fun.py b/fun.py index cbc37b2f0..1466a78bb 100644 --- a/fun.py +++ b/fun.py @@ -106,6 +106,11 @@ def msb_size(data, offset=0): raise AssertionError("Could not find terminating MSB byte in data stream") return i+offset, size +def loose_object_header(type, size): + """:return: string representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return "%s %i\0" % (type, size) + def write_object(type, size, read, write, chunk_size=chunk_size): """Write the object as identified by type, size and source_stream into the target_stream @@ -120,7 +125,7 @@ def write_object(type, size, read, write, chunk_size=chunk_size): tbw = 0 # total num bytes written # WRITE HEADER: type SP size NULL - tbw += write("%s %i\0" % (type, size)) + tbw += write(loose_object_header(type, size)) tbw += stream_copy(read, write, size, chunk_size) return tbw diff --git a/pack.py b/pack.py index 811acaf98..b1f2f1e05 100644 --- a/pack.py +++ b/pack.py @@ -3,6 +3,7 @@ BadObject, ) from util import ( + zlib, LockedFD, LazyMixin, unpack_from, @@ -12,6 +13,7 @@ from fun import ( pack_object_header_info, type_id_to_type_map, + write_object, stream_copy, chunk_size, delta_types, @@ -31,6 +33,7 @@ from stream import ( DecompressMemMapReader, DeltaApplyReader, + Sha1Writer, NullStream, ) @@ -38,7 +41,8 @@ pack, ) -__all__ = ('PackIndexFile', 'PackFile') +import os +__all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -237,6 +241,10 @@ def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._data[-40:-20] @@ -288,8 +296,8 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 # header bytes - _footer_size = 20 # final sha + first_object_offset = 3*4 # header bytes + footer_size = 20 # final sha def __init__(self, packpath): self._packpath = packpath @@ -312,8 +320,8 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data - content_size = len(data) - self._footer_size - cur_offset = start_offset or self._first_object_offset + content_size = len(data) - self.footer_size + cur_offset = start_offset or self.first_object_offset null = NullStream() while cur_offset < content_size: @@ -343,10 +351,18 @@ def version(self): """:return: the version of this pack""" return self._version + def data(self): + """:return: read-only data of this pack. It provides random access and usually + is a memory map""" + return self._data + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._data[-20:] - + + def path(self): + """:return: path to the packfile""" + return self._packpath #} END pack information #{ Pack Specific @@ -383,13 +399,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self._first_object_offset, False) + return pack_object_at(self._data, offset or self.first_object_offset, False) def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self._first_object_offset, True) + return pack_object_at(self._data, offset or self.first_object_offset, True) def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing @@ -403,7 +419,7 @@ def stream_iter(self, start_offset=0): #} END Read-Database like Interface -class PackFileEntity(object): +class PackEntity(object): """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" @@ -412,11 +428,12 @@ class PackFileEntity(object): IndexFileCls = PackIndexFile PackFileCls = PackFile - def __init__(self, basename): + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) @@ -426,12 +443,20 @@ def _sha_to_index(self, sha): def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" - raise NotImplementedError() - - def _object(self, sha, as_stream): - """:return: OInfo or OStream object providing information about the given sha""" + indexfile = self._index + _object = self._object + for index in xrange(indexfile.size()): + sha = indexfile.sha(index) + yield _object(sha, as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" # its a little bit redundant here, but it needs to be efficient - offset = self._index.offset(self._sha_to_index(sha)) + if index < 0: + index = self._sha_to_index(sha) + offset = self._index.offset(index) type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) if as_stream: if type_id not in delta_types: @@ -447,7 +472,7 @@ def _object(self, sha, as_stream): offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - streams[0].seek(0) # assure it can be read by the delta reader + streams[0].stream.seek(0) # assure it can be read by the delta reader dstream = DeltaApplyReader.new(streams) return OStream(sha, dstream.type, target_size, dstream) @@ -476,20 +501,79 @@ def info(self, sha): """Retrieve information about the object identified by the given sha :param sha: 20 byte sha1 :raise BadObject: - :return: OInfo instance""" + :return: OInfo instance, with 20 byte sha""" return self._object(sha, as_stream=False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: - :return: OStream instance""" + :return: OStream instance, with 20 byte sha""" return self._object(sha, as_stream=True) #} END Read-Database like Interface #{ Interface - + + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """Verify that the stream at the given sha is valid. + :param sha: 20 byte sha1 of the object whose stream to verify + :param use_crc: if True, the index' crc for the sha is used to determine + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + index = self._sha_to_index(sha) + offset = self._index.offset(index) + pack_data = self._pack.data() + next_index = min(self._index.size()-1, index+1) + next_offset = 0 + if next_index == index: + next_offset = len(pack_data) - self._pack.footer_size + else: + next_offset = self._index.offset(next_index) + # END get next offset + crc_value = self._index.crc(index) + + this_crc_value = 0 + crc_update = zlib.crc32 + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + cur_pos = offset + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + cur_pos += size + # END window size loop + + assert this_crc_value == crc_value + return this_crc_value == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + return True + def info_iter(self): """:return: Iterator over all objects in this pack. The iterator yields OInfo instances""" diff --git a/stream.py b/stream.py index 10a8e057c..903aeb109 100644 --- a/stream.py +++ b/stream.py @@ -474,7 +474,7 @@ class Sha1Writer(object): __slots__ = "sha1" def __init__(self): - self.sha1 = make_sha("") + self.sha1 = make_sha() #{ Stream Interface diff --git a/test/test_pack.py b/test/test_pack.py index 1f860961e..03b162178 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -10,10 +10,16 @@ ) from gitdb.pack import ( + PackEntity, PackIndexFile, PackFile ) + +from gitdb.fun import ( + delta_types, + ) from gitdb.util import to_bin_sha +from itertools import izip import os @@ -84,7 +90,7 @@ def _assert_pack_file(self, pack, version, size): # END get deltastream # read all - assert len(dstream.read()) + assert len(dstream.read()) == dstream.size # read chunks # NOTE: the current implementation is safe, it basically transfers @@ -109,8 +115,34 @@ def test_pack(self): # END for each pack to test def test_pack_entity(self): - # TODO: - pass + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + print packfile + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + + count = 0 + for info, stream in izip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.sha == stream.sha + assert len(info.sha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # verify the stream + print info + assert entity.is_valid_stream(info.sha, use_crc=True) + #assert entity.is_valid_stream(info.sha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets diff --git a/util.py b/util.py index f10b71b12..c460969ca 100644 --- a/util.py +++ b/util.py @@ -79,21 +79,6 @@ def make_sha(source=''): sha1 = sha.sha(source) return sha1 -def stream_copy(source, destination, chunk_size=512*1024): - """Copy all data from the source stream into the destination stream in chunks - of size chunk_size - - :return: amount of bytes written""" - br = 0 - while True: - chunk = source.read(chunk_size) - destination.write(chunk) - br += len(chunk) - if len(chunk) < chunk_size: - break - # END reading output stream - return br - def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" try: From ecb18782c423bfdc45a474af2b7fbb61f62fa750 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 17:56:20 +0200 Subject: [PATCH 0024/3719] CRC verification already works for all packs, sha1 still needs some work, probably with deltified objects, there it shows whether we did it aaaaaall correctly ;) --- pack.py | 75 ++++++++++++++++++++++++++++++++++++----------- test/test_pack.py | 14 +++++---- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/pack.py b/pack.py index b1f2f1e05..4003e2742 100644 --- a/pack.py +++ b/pack.py @@ -1,6 +1,7 @@ """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, + BadObject, + UnsupportedOperation ) from util import ( zlib, @@ -41,6 +42,8 @@ pack, ) +from itertools import izip +import array import os __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -253,6 +256,21 @@ def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._data[-20:] + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + :note: return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) + + # networkbyteorder to something array likes more + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in xrange(self.size())) + # END handle version + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -419,11 +437,14 @@ def stream_iter(self, start_offset=0): #} END Read-Database like Interface -class PackEntity(object): +class PackEntity(LazyMixin): """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - __slots__ = ('_index', '_pack') + __slots__ = ( '_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) IndexFileCls = PackIndexFile PackFileCls = PackFile @@ -433,6 +454,28 @@ def __init__(self, pack_or_index_path): basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def _set_cache_(self, attr): + # currently this can only be _offset_map + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = { offsets_sorted[0] : last_offset } + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + iter_offsets_plus_one.next() + consecutive = izip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" @@ -537,33 +580,31 @@ def is_valid_stream(self, sha, use_crc=False): :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + index = self._sha_to_index(sha) offset = self._index.offset(index) - pack_data = self._pack.data() - next_index = min(self._index.size()-1, index+1) - next_offset = 0 - if next_index == index: - next_offset = len(pack_data) - self._pack.footer_size - else: - next_offset = self._index.offset(next_index) - # END get next offset + next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - this_crc_value = 0 - crc_update = zlib.crc32 - # create the current crc value, on the compressed object data # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() cur_pos = offset + this_crc_value = 0 while cur_pos < next_offset: rbound = min(cur_pos + chunk_size, next_offset) size = rbound - cur_pos - crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - assert this_crc_value == crc_value - return this_crc_value == crc_value + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value else: shawriter = Sha1Writer() stream = self._object(sha, as_stream=True) diff --git a/test/test_pack.py b/test/test_pack.py index 03b162178..d9823efa0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -15,9 +15,8 @@ PackFile ) -from gitdb.fun import ( - delta_types, - ) +from gitdb.fun import delta_types +from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha from itertools import izip import os @@ -42,6 +41,7 @@ def _assert_index_file(self, index, version, size): assert len(index.indexfile_checksum()) == 20 assert index.version() == version assert index.size() == size + assert len(index.offsets()) == size # get all data of all objects for oidx in xrange(index.size()): @@ -137,8 +137,12 @@ def test_pack_entity(self): # verify the stream print info - assert entity.is_valid_stream(info.sha, use_crc=True) - #assert entity.is_valid_stream(info.sha, use_crc=False) + try: + assert entity.is_valid_stream(info.sha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.sha, use_crc=False) # END for each info, stream tuple assert count == size From 5af5cd919aea64aeb8be8a5dc38cc6a169878399 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 18:36:33 +0200 Subject: [PATCH 0025/3719] Sha1 verification works as well, forgot to fill in the base buffer for delta-application, and fixed the broken DeltaApplyReader's seek method --- fun.py | 2 +- pack.py | 1 + stream.py | 6 +++++- test/test_pack.py | 10 +++++++--- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fun.py b/fun.py index 1466a78bb..dd9a53b5d 100644 --- a/fun.py +++ b/fun.py @@ -200,7 +200,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if (cp_off + cp_size < cp_size or cp_off + cp_size > src_buf_size): break - twrite(src_buf[cp_off:cp_off+cp_size]) + twrite(buffer(src_buf, cp_off, cp_size)) elif c: twrite(db[i:i+c]) i += c diff --git a/pack.py b/pack.py index 4003e2742..ca9dac959 100644 --- a/pack.py +++ b/pack.py @@ -611,6 +611,7 @@ def is_valid_stream(self, sha, use_crc=False): # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification return True diff --git a/stream.py b/stream.py index 903aeb109..b0fa4acb5 100644 --- a/stream.py +++ b/stream.py @@ -358,6 +358,7 @@ def _set_cache_(self, attr): # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final @@ -408,6 +409,8 @@ def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying data = self._mm_target.read(count) self._br += len(data) return data @@ -418,7 +421,8 @@ def seek(self, offset, whence=os.SEEK_SET): if offset != 0 or whence != os.SEEK_SET: raise ValueError("Can only seek to position 0") # END handle offset - self._size + self._br = 0 + self._mm_target.seek(0) #{ Interface diff --git a/test/test_pack.py b/test/test_pack.py index d9823efa0..fbcf84e39 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -90,7 +90,13 @@ def _assert_pack_file(self, pack, version, size): # END get deltastream # read all - assert len(dstream.read()) == dstream.size + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + # read chunks # NOTE: the current implementation is safe, it basically transfers @@ -119,7 +125,6 @@ def test_pack_entity(self): (self.packfile_v2_2, self.packindexfile_v2)): packfile, version, size = packinfo indexfile, version, size = indexinfo - print packfile entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile @@ -136,7 +141,6 @@ def test_pack_entity(self): assert not info.type_id in delta_types # verify the stream - print info try: assert entity.is_valid_stream(info.sha, use_crc=True) except UnsupportedOperation: From 325742cfe258436302fe0bb92462dc18a22261c7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 22:36:44 +0200 Subject: [PATCH 0026/3719] Implemented basic info and stream retrieval as well as pack file handling of PackedDB - its now operational. Next up is a performance test --- db/pack.py | 123 +++++++++++++++++++++++++++++++++++++++++-- pack.py | 30 +++++++---- test/db/test_pack.py | 45 ++++++++++++++-- test/test_pack.py | 14 +++-- 4 files changed, 193 insertions(+), 19 deletions(-) diff --git a/db/pack.py b/db/pack.py index a850e0fb2..f5ee04d6e 100644 --- a/db/pack.py +++ b/db/pack.py @@ -4,29 +4,95 @@ ObjectDBR ) +from gitdb.util import ( + to_bin_sha, + LazyMixin + ) + from gitdb.exc import ( + BadObject, UnsupportedOperation, ) +from gitdb.pack import PackEntity + +import os +import glob __all__ = ('PackedDB', ) -class PackedDB(FileDBBase, ObjectDBR): + +#{ Utilities + + +class PackedDB(FileDBBase, ObjectDBR, LazyMixin): """A database operating on a set of object packs""" + # sort the priority list every N queries + _sort_interval = 15 + def __init__(self, root_path): super(PackedDB, self).__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + # currently it can only be our _entities attribute + self._entities = list() + self.update_pack_entity_cache() + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + :note: This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + sha = to_bin_sha(sha) + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) #{ Object DB Read def has_object(self, sha): - raise NotImplementedError() + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling def info(self, sha): - raise NotImplementedError() + entity, index = self._pack_info(sha) + return entity.info_at_index(index) def stream(self, sha): - raise NotImplementedError() + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) #} END object db read @@ -39,6 +105,55 @@ def store(self, istream): raise UnsupportedOperation() def store_async(self, reader): + # TODO: add ObjectDBRW before implementing this raise NotImplementedError() #} END object db write + + + #{ Interface + + def update_pack_entity_cache(self, force=False): + """Update our cache with the acutally existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = set(item[1].pack().path() for item in self._entities) + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + #} END interface diff --git a/pack.py b/pack.py index ca9dac959..4e70c7ce9 100644 --- a/pack.py +++ b/pack.py @@ -40,6 +40,7 @@ from struct import ( pack, + unpack, ) from itertools import izip @@ -196,7 +197,7 @@ def _offset_v2(self, i): # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: - offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0] + offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset return offset @@ -291,7 +292,7 @@ def sha_to_index(self, sha): elif not c: return mid else: - lo = mid + lo = mid + 1 # END handle midpoint # END bisect return None @@ -326,13 +327,15 @@ def _set_cache_(self, attr): fd = ldb.open() self._data = file_contents_ro(fd) ldb.rollback() - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - else: + # read the header information type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) - assert type_id == "PACK", "Pack file format is invalid: %r" % type_id - assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + else: # must be '_size' or '_version' + # read header info - we do that just with a file stream + type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12)) # END handle header def _iter_objects(self, start_offset, as_stream=True): @@ -545,14 +548,23 @@ def info(self, sha): :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" - return self._object(sha, as_stream=False) + return self._object(sha, False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" - return self._object(sha, as_stream=True) + return self._object(sha, True) + + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) #} END Read-Database like Interface diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 6faff4695..d37c886eb 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -1,12 +1,51 @@ from lib import * from gitdb.db import PackedDB - +from gitdb.test.lib import fixture_path + +import os +import random + class TestPackDB(TestDBBase): @with_rw_directory @with_packs_rw def test_writing(self, path): - ldb = PackedDB(path) - # TODO + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = 2 + assert len(pdb._entities) == num_packs + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb._entities[0][1].pack().path() + new_pack_path = pack_path + "renamed" + os.rename(pack_path, new_pack_path) + pdb.update_pack_entity_cache(force=True) + assert len(pdb._entities) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_pack_entity_cache(force=True) + assert len(pdb._entities) == num_packs + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list() + for entity in (item[1] for item in pdb._entities): + for index in xrange(entity.index().size()): + + sha_list.append(entity.index().sha(index)) + # END for each index + # END for each entity + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + info = pdb.info(sha) + stream = pdb.stream(sha) + # END for each sha to query diff --git a/test/test_pack.py b/test/test_pack.py index fbcf84e39..6cfd784d4 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,9 +5,7 @@ with_packs_rw, fixture_path ) -from gitdb.stream import ( - DeltaApplyReader - ) +from gitdb.stream import DeltaApplyReader from gitdb.pack import ( PackEntity, @@ -15,6 +13,11 @@ PackFile ) +from gitdb.base import ( + OInfo, + OStream, + ) + from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha @@ -140,6 +143,11 @@ def test_pack_entity(self): # we return fully resolved items, which is implied by the sha centric access assert not info.type_id in delta_types + # try all calls + assert len(entity.collect_streams(info.sha)) + assert isinstance(entity.info(info.sha), OInfo) + assert isinstance(entity.stream(info.sha), OStream) + # verify the stream try: assert entity.is_valid_stream(info.sha, use_crc=True) From 8ab9b4f307a0672fb7b7ae29812b4ffd3d9dc8bc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 23:36:53 +0200 Subject: [PATCH 0027/3719] PackedDB: added sha_iter and size methods, these should move to the ObjectDBR actually Added performance test, packed stream reading still runs into errors, which is interesting as it dealt with the sample packs very well before --- db/pack.py | 19 +++++++++- test/db/test_pack.py | 9 ++--- test/performance/lib.py | 1 + test/performance/test_db.py | 70 +++++++++++++++++++++++++++++++++---- 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/db/pack.py b/db/pack.py index f5ee04d6e..97890ee20 100644 --- a/db/pack.py +++ b/db/pack.py @@ -28,7 +28,9 @@ class PackedDB(FileDBBase, ObjectDBR, LazyMixin): """A database operating on a set of object packs""" # sort the priority list every N queries - _sort_interval = 15 + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 def __init__(self, root_path): super(PackedDB, self).__init__(root_path) @@ -156,4 +158,19 @@ def update_pack_entity_cache(self, force=False): self._sort_entities() return True + def sha_iter(self): + """Return iterator yielding 20 byte shas for the packed objects in this data base""" + sha_list = list() + for entity in (item[1] for item in self._entities): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + """:return: amount of packed objects in this database""" + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes) #} END interface diff --git a/test/db/test_pack.py b/test/db/test_pack.py index d37c886eb..89f73f036 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -34,13 +34,8 @@ def test_writing(self, path): # bang on the cache # access the Entities directly, as there is no iteration interface # yet ( or required for now ) - sha_list = list() - for entity in (item[1] for item in pdb._entities): - for index in xrange(entity.index().size()): - - sha_list.append(entity.index().sha(index)) - # END for each index - # END for each entity + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() # hit all packs in random order random.shuffle(sha_list) diff --git a/test/performance/lib.py b/test/performance/lib.py index 03788c081..45e0ca53f 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -44,6 +44,7 @@ def setUpAll(cls): except AttributeError: pass cls.gitrepopath = resolve_or_fail(k_env_git_repo) + assert cls.gitrepopath.endswith('.git') #} END base classes diff --git a/test/performance/test_db.py b/test/performance/test_db.py index cd231b650..f6d855e20 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -1,15 +1,71 @@ """Performance tests for object store""" +from lib import ( + TestBigRepoR + ) + +from gitdb.db.pack import PackedDB import sys +import os from time import time - -from lib import ( - TestBigRepoR - ) +import random class TestGitDBPerformance(TestBigRepoR): - def test_random_access(self): - pass - # TODO: use the actual db for this + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + assert len(pdb._entities) > 1 + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + access_times = list() + for rand in range(2): + if rand: + random.shuffle(sha_list) + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + access_times.append(elapsed) + + # discard cache + del(pdb._entities) + pdb._entities + print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) + # END for each random mode + elapsed_order, elapsed_rand = access_times + + # well, its never really sequencial regarding the memory patterns, but it + # shows how well the prioriy cache performs + print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100)) + + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + # END for each function + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + stream.read() + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed) From e9c5cf3df54d0879662194f22d43a984e89b2cdf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 00:49:34 +0200 Subject: [PATCH 0028/3719] delta-apply now works after fixing a stupid type, instead of i + 1 I wrote i + i ... argh \! --- fun.py | 18 +++++++++++++----- stream.py | 24 +++++++++++------------- test/performance/test_db.py | 5 ++++- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/fun.py b/fun.py index dd9a53b5d..82b9373b2 100644 --- a/fun.py +++ b/fun.py @@ -155,7 +155,8 @@ def stream_copy(read, write, size, chunk_size): return dbw -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, + target_size): """Apply data from a delta buffer using a source buffer to the target file, which will be written to :param src_buf: random access data from which the delta was created @@ -163,6 +164,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param target_file: file like object to write the result to + :param target_size: size of the target buffer :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 twrite = target_file.write @@ -180,7 +182,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi i += 1 if (c & 0x04): cp_off |= (ord(db[i]) << 16) - i += i + i += 1 if (c & 0x08): cp_off |= (ord(db[i]) << 24) i += 1 @@ -196,14 +198,20 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if not cp_size: cp_size = 0x10000 - # maybe skip this check ? - if (cp_off + cp_size < cp_size or - cp_off + cp_size > src_buf_size): + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size or + cp_size > target_size): break twrite(buffer(src_buf, cp_off, cp_size)) + target_size -= cp_size elif c: + if c > target_size: + break twrite(db[i:i+c]) i += c + target_size -= c else: raise ValueError("unexpected delta opcode 0") # END handle command byte diff --git a/stream.py b/stream.py index b0fa4acb5..8c171b23a 100644 --- a/stream.py +++ b/stream.py @@ -327,19 +327,15 @@ def __init__(self, stream_list): def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" - # fill in delta info structures, providing the source and target buffer - # sizes. - buffer_offset_list = list() - final_target_size = None + + # prefetch information + buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: buf = dstream.read(512) # read the header information + X offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - if final_target_size is None: - final_target_size = target_size - # END set final target size - buffer_offset_list.append((buffer(buf, offset), offset)) + buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream @@ -358,7 +354,7 @@ def _set_cache_(self, attr): # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final @@ -370,7 +366,8 @@ def _set_cache_(self, attr): # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. - for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)): + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) @@ -381,15 +378,16 @@ def _set_cache_(self, attr): # read the rest from the stream. The size we give is larger than necessary stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - ################################################################ - apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf) - ################################################################ + ####################################################################### + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size) + ####################################################################### # finally, swap out source and target buffers. The target is now the # base for the next delta to apply bbuf, tbuf = tbuf, bbuf bbuf.seek(0) tbuf.seek(0) + final_target_size = target_size # END for each delta to apply # its already seeked to 0, constrain it to the actual size diff --git a/test/performance/test_db.py b/test/performance/test_db.py index f6d855e20..3948003cc 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -63,9 +63,12 @@ def test_pack_random_access(self): # retrieve stream and read all max_items = 5000 pdb_stream = pdb.stream + total_size = 0 st = time() for sha in sha_list[:max_items]: stream = pdb_stream(sha) stream.read() + total_size += stream.size elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed) + total_kib = total_size / 1000 + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) From fc6253d8428631029a9cb42830c9829692e92997 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 01:04:45 +0200 Subject: [PATCH 0029/3719] removed some extra checks in apply-delta which are indeed not required --- fun.py | 11 ++--------- stream.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/fun.py b/fun.py index 82b9373b2..d7e43717c 100644 --- a/fun.py +++ b/fun.py @@ -155,8 +155,7 @@ def stream_copy(read, write, size, chunk_size): return dbw -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, - target_size): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): """Apply data from a delta buffer using a source buffer to the target file, which will be written to :param src_buf: random access data from which the delta was created @@ -164,7 +163,6 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param target_file: file like object to write the result to - :param target_size: size of the target buffer :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 twrite = target_file.write @@ -201,17 +199,12 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size or - cp_size > target_size): + rbound > src_buf_size): break twrite(buffer(src_buf, cp_off, cp_size)) - target_size -= cp_size elif c: - if c > target_size: - break twrite(db[i:i+c]) i += c - target_size -= c else: raise ValueError("unexpected delta opcode 0") # END handle command byte diff --git a/stream.py b/stream.py index 8c171b23a..6c388a96c 100644 --- a/stream.py +++ b/stream.py @@ -379,7 +379,7 @@ def _set_cache_(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size) + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf) ####################################################################### # finally, swap out source and target buffers. The target is now the From ae6d08e1b63bf05ddcb3ee5fe027c5d829380afc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 10:10:28 +0200 Subject: [PATCH 0030/3719] Removed data-offset field from PackInfo as it is not needed in most cases. Instead, pack_at_offset returns the data-offset, slightly improving performance, and reducing memory demands --- base.py | 36 ++++++++++++++++-------------------- pack.py | 23 ++++++++++++----------- test/performance/test_db.py | 2 +- test/test_base.py | 2 -- test/test_pack.py | 1 - 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/base.py b/base.py index aa917858d..25968f371 100644 --- a/base.py +++ b/base.py @@ -64,8 +64,8 @@ class OPackInfo(tuple): location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size): - return tuple.__new__(cls, (packoffset, dataoffset, type, size)) + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset,type, size)) def __init__(self, *args): tuple.__init__(self) @@ -76,21 +76,17 @@ def __init__(self, *args): def pack_offset(self): return self[0] - @property - def data_offset(self): - return self[1] - @property def type(self): - return type_id_to_type_map[self[2]] + return type_id_to_type_map[self[1]] @property def type_id(self): - return self[2] + return self[1] @property def size(self): - return self[3] + return self[2] #} END interface @@ -102,13 +98,13 @@ class ODeltaPackInfo(OPackInfo): the pack offset of the base object""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, delta_info): - return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info)) + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) #{ Interface @property def delta_info(self): - return self[4] + return self[3] #} END interface @@ -142,17 +138,17 @@ class OPackStream(OPackInfo): is provided""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, stream, *args): + def __new__(cls, packoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream)) + return tuple.__new__(cls, (packoffset, type, size, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[4].read(size) + return self[3].read(size) @property def stream(self): - return self[4] + return self[3] #} END stream reader interface @@ -160,17 +156,17 @@ class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream): - return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream)) + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[5].read(size) + return self[4].read(size) @property def stream(self): - return self[5] + return self[4] #} END stream reader interface diff --git a/pack.py b/pack.py index 4e70c7ce9..1996d0389 100644 --- a/pack.py +++ b/pack.py @@ -55,7 +55,7 @@ def pack_object_at(data, offset, as_stream): """ - :return: PackInfo|PackStream + :return: Tuple(abs_data_offset, PackInfo|PackStream) an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. @@ -97,14 +97,14 @@ def pack_object_at(data, offset, as_stream): if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) if delta_info is None: - return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream) + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream) + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: - return OPackInfo(offset, abs_data_offset, type_id, uncomp_size) + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) else: - return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info) + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) # END handle info # END handle stream @@ -278,6 +278,7 @@ def sha_to_index(self, sha): if the sha was not found in this pack index :param sha: 20 byte sha to lookup""" first_byte = ord(sha[0]) + get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] @@ -286,7 +287,7 @@ def sha_to_index(self, sha): # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 - c = cmp(sha, self.sha(mid)) + c = cmp(sha, get_sha(mid)) if c < 0: hi = mid elif not c: @@ -346,12 +347,12 @@ def _iter_objects(self, start_offset, as_stream=True): null = NullStream() while cur_offset < content_size: - ostream = pack_object_at(data, cur_offset, True) + data_offset, ostream = pack_object_at(data, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand @@ -399,7 +400,7 @@ def collect_streams(self, offset): :param offset: specifies the first byte of the object within this pack""" out = list() while True: - ostream = pack_object_at(self._data, offset, True) + ostream = pack_object_at(self._data, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -420,13 +421,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, False) + return pack_object_at(self._data, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, True) + return pack_object_at(self._data, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing diff --git a/test/performance/test_db.py b/test/performance/test_db.py index 3948003cc..46db794fd 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -14,7 +14,6 @@ class TestGitDBPerformance(TestBigRepoR): def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - assert len(pdb._entities) > 1 # sha lookup st = time() @@ -72,3 +71,4 @@ def test_pack_random_access(self): elapsed = time() - st total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + diff --git a/test/test_base.py b/test/test_base.py index 524bf3054..c122ec4b1 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -35,14 +35,12 @@ def test_streams(self): assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - assert pinfo.data_offset == 1 dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - assert dpinfo.data_offset == 1 # test ostream diff --git a/test/test_pack.py b/test/test_pack.py index 6cfd784d4..6821097ad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -72,7 +72,6 @@ def _assert_pack_file(self, pack, version, size): stream = pack.stream(obj.pack_offset) assert info.pack_offset == stream.pack_offset - assert info.data_offset == stream.data_offset assert info.type_id == stream.type_id assert hasattr(stream, 'read') From 4503a78e699cb42935f15baa46f5947a0020911f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 12:37:42 +0200 Subject: [PATCH 0031/3719] Added initial test for putting some often-called functions into an extension module, for now its only being built by a cheap hardcoded makefile. It shows that the performance gain is rather small, bottlenecks are attr accesses, so in fact the whole type wants to be put into C to get real performance. Its not really worth it for 25% I believe --- .gitignore | 2 ++ _fun.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ makefile | 12 +++++++ pack.py | 13 ++++++++ 4 files changed, 124 insertions(+) create mode 100644 _fun.c create mode 100644 makefile diff --git a/.gitignore b/.gitignore index 0d20b6487..1a9d961a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.pyc +*.o +*.so diff --git a/_fun.c b/_fun.c new file mode 100644 index 000000000..ce9f25b16 --- /dev/null +++ b/_fun.c @@ -0,0 +1,97 @@ +#include +#include + +static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) +{ + const unsigned char *sha; + const unsigned int sha_len; + + // Note: self is only set if we are a c type. We emulate an instance method, + // hence we have to get the instance as 'first' argument + + // get instance and sha + PyObject* inst = 0; + if (!PyArg_ParseTuple(args, "Os#", &inst, &sha, &sha_len)) + return NULL; + + if (sha_len != 20) { + PyErr_SetString(PyExc_ValueError, "Sha is not 20 bytes long"); + return NULL; + } + + if( !inst){ + PyErr_SetString(PyExc_ValueError, "Cannot be called without self"); + return NULL; + } + + // read lo and hi bounds + PyObject* fanout_table = PyObject_GetAttrString(inst, "_fanout_table"); + if (!fanout_table){ + PyErr_SetString(PyExc_ValueError, "Couldn't obtain fanout table"); + return NULL; + } + + unsigned int lo = 0, hi = 0; + if (sha[0]){ + PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)(sha[0]-1)); + lo = PyInt_AS_LONG(item); + Py_DECREF(item); + } + PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)sha[0]); + hi = PyInt_AS_LONG(item); + Py_DECREF(item); + item = 0; + + Py_DECREF(fanout_table); + + // get sha query function + PyObject* get_sha = PyObject_GetAttrString(inst, "sha"); + if (!get_sha){ + PyErr_SetString(PyExc_ValueError, "Couldn't obtain sha method"); + return NULL; + } + + PyObject *sha_str = 0; + while (lo < hi) { + const int mid = (lo + hi)/2; + sha_str = PyObject_CallFunction(get_sha, "i", mid); + if (!sha_str) { + return NULL; + } + + // we really trust that string ... for speed + const int cmp = memcmp(PyString_AS_STRING(sha_str), sha, 20); + Py_DECREF(sha_str); + sha_str = 0; + + if (cmp < 0){ + lo = mid + 1; + } + else if (cmp > 0) { + hi = mid; + } + else { + Py_DECREF(get_sha); + return PyInt_FromLong(mid); + }// END handle comparison + }// END while lo < hi + + // nothing found, cleanup + Py_DECREF(get_sha); + Py_RETURN_NONE; +} + + +static PyMethodDef py_fun[] = { + { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL }, + { NULL, NULL, 0, NULL } +}; + +void init_fun(void) +{ + PyObject *m; + + m = Py_InitModule3("_fun", py_fun, NULL); + if (m == NULL) + return; +} diff --git a/makefile b/makefile new file mode 100644 index 000000000..390289625 --- /dev/null +++ b/makefile @@ -0,0 +1,12 @@ + +_fun.o: _fun.c + gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@ + +_fun.so: _fun.o + gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@ + +all: _fun.so + +clean: + -rm *.so + -rm *.o diff --git a/pack.py b/pack.py index 1996d0389..d6fe56823 100644 --- a/pack.py +++ b/pack.py @@ -23,6 +23,12 @@ msb_size ) +try: + from _fun import PackIndexFile_sha_to_index +except ImportError: + pass +# END try c module + from base import ( # Amazing ! OInfo, OStream, @@ -298,6 +304,13 @@ def sha_to_index(self, sha): # END bisect return None + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + #} END properties From 3cee78ed377b0a73febdbd772ddba2999313023e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 15:52:16 +0200 Subject: [PATCH 0032/3719] Added endurance run to check all objects in the git source repository, something like a primtive git-fsck, which indeed takes a while to run. Its useful to see the memory consumption, which must stay static in the domain of physical memory --- db/pack.py | 6 ++++- test/performance/{test_db.py => test_pack.py} | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) rename test/performance/{test_db.py => test_pack.py} (74%) diff --git a/db/pack.py b/db/pack.py index 97890ee20..2332992f2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -157,11 +157,15 @@ def update_pack_entity_cache(self, force=False): # reinitialize prioritiess self._sort_entities() return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [ item[1] for item in self._entities ] def sha_iter(self): """Return iterator yielding 20 byte shas for the packed objects in this data base""" sha_list = list() - for entity in (item[1] for item in self._entities): + for entity in self.entities(): index = entity.index() sha_by_index = index.sha for index in xrange(index.size()): diff --git a/test/performance/test_db.py b/test/performance/test_pack.py similarity index 74% rename from test/performance/test_db.py rename to test/performance/test_pack.py index 46db794fd..37abc1725 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_pack.py @@ -10,7 +10,7 @@ from time import time import random -class TestGitDBPerformance(TestBigRepoR): +class TestPackedDBPerformance(TestBigRepoR): def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -22,7 +22,6 @@ def test_pack_random_access(self): ns = len(sha_list) print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) - # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info access_times = list() @@ -39,7 +38,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) - pdb._entities + pdb.entities() print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) # END for each random mode elapsed_order, elapsed_rand = access_times @@ -72,3 +71,24 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + + print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in xrange(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + except UnsupportedOperation: + pass + # END ignore old indices + count += 1 + # END for each index + # END for each entity + elapsed = time() - st + print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + # END for each verify mode + From 6fd8d7406028d603379dc23be0ab1403785f2cd3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 19:57:28 +0200 Subject: [PATCH 0033/3719] Base implementation and stubs added for git-like db, as well as the reference db ( for the alternates implementation ) --- db/base.py | 73 +++++++++++++++++++++++++++++++++-- db/git.py | 68 +++++++++++++++++++------------- db/loose.py | 19 +++++++++ db/pack.py | 45 ++++++++++----------- db/ref.py | 61 ++++++++++++++++++++++++++++- test/db/lib.py | 1 + test/db/test_git.py | 9 +++++ test/db/test_loose.py | 5 +++ test/db/test_pack.py | 12 +++--- test/db/test_ref.py | 21 ++++++++++ test/performance/test_pack.py | 5 ++- util.py | 1 + 12 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 test/db/test_git.py create mode 100644 test/db/test_ref.py diff --git a/db/base.py b/db/base.py index 2cda0ea0a..91e82fc0b 100644 --- a/db/base.py +++ b/db/base.py @@ -9,7 +9,7 @@ ) -__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB') +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') class ObjectDBR(object): @@ -66,6 +66,14 @@ def stream_async(self, reader): # base implementation just uses the stream method repeatedly task = ChannelThreadTask(reader, str(self.stream_async), self.stream) return pool.add_task(task) + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() #} END query interface @@ -150,6 +158,63 @@ def db_path(self, rela_path): #} END interface -class CompoundDB(ObjectDBR): - """A database which delegates calls to sub-databases""" - # TODO +class CachingDB(object): + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """Call this method if the underlying data changed to trigger an update + of the internal caching structures. + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface + + +class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + + #{ ObjectDBR interface + + def has_object(self, sha): + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + raise NotImplementedError() + + def size(self): + raise NotImplementedError() + + def sha_iter(self): + raise NotImplementedError() + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) + + def update_cache(self, force=False): + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + #} END interface + + diff --git a/db/git.py b/db/git.py index 0488bc601..1953def17 100644 --- a/db/git.py +++ b/db/git.py @@ -1,33 +1,49 @@ - -from gitdb.base import ( - OInfo, - OStream - ) +from base import ( + CompoundDB, + FileDBBase, + ) from loose import LooseObjectDB +from pack import PackedDB +from ref import ReferenceDB + +from gitdb.util import LazyMixin +from gitdb.exc import InvalidDBRoot +import os -__all__ = ('GitObjectDB', ) +__all__ = ('GitDB', ) -#class GitObjectDB(CompoundDB, ObjectDBW): -class GitObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file +class GitDB(FileDBBase, CompoundDB): + """A git-style object database, which contains all objects in the 'objects' + subdirectory""" + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB - It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations - """ - def __init__(self, root_path, git): - """Initialize this instance with the root and a git command""" - super(GitObjectDB, self).__init__(root_path) - self._git = git + # Directories + packs_dir = 'packs' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super(GitDB, self).__init__(root_path) - def info(self, sha): - t = self._git.get_object_header(sha) - return OInfo(*t) + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle dbs - def stream(self, sha): - """For now, all lookup is done by git itself""" - t = self._git.stream_object_data(sha) - return OStream(*t) - diff --git a/db/loose.py b/db/loose.py index 109782fdc..b95d6f1c9 100644 --- a/db/loose.py +++ b/db/loose.py @@ -24,11 +24,13 @@ from gitdb.util import ( ENOENT, to_hex_sha, + hex_to_bin, exists, isdir, mkdir, rename, dirname, + basename, join ) @@ -186,4 +188,21 @@ def store(self, istream): istream.sha = sha return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) diff --git a/db/pack.py b/db/pack.py index 2332992f2..92fbc616b 100644 --- a/db/pack.py +++ b/db/pack.py @@ -1,7 +1,8 @@ """Module containing a database to deal with packs""" from base import ( FileDBBase, - ObjectDBR + ObjectDBR, + CachingDB ) from gitdb.util import ( @@ -18,13 +19,13 @@ import os import glob -__all__ = ('PackedDB', ) +__all__ = ('PackedDB', ) #{ Utilities -class PackedDB(FileDBBase, ObjectDBR, LazyMixin): +class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): """A database operating on a set of object packs""" # sort the priority list every N queries @@ -43,9 +44,10 @@ def __init__(self, root_path): self._st_mtime = 0 # last modification data of our root path def _set_cache_(self, attr): - # currently it can only be our _entities attribute - self._entities = list() - self.update_pack_entity_cache() + if attr == '_entities': + self._entities = list() + self.update_cache() + # END handle entities initialization def _sort_entities(self): self._entities.sort(key=lambda l: l[0], reverse=True) @@ -95,6 +97,20 @@ def info(self, sha): def stream(self, sha): entity, index = self._pack_info(sha) return entity.stream_at_index(index) + + def sha_iter(self): + sha_list = list() + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes) #} END object db read @@ -115,7 +131,7 @@ def store_async(self, reader): #{ Interface - def update_pack_entity_cache(self, force=False): + def update_cache(self, force=False): """Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones :param force: If True, the cache will be updated even though the directory @@ -162,19 +178,4 @@ def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - def sha_iter(self): - """Return iterator yielding 20 byte shas for the packed objects in this data base""" - sha_list = list() - for entity in self.entities(): - index = entity.index() - sha_by_index = index.sha - for index in xrange(index.size()): - yield sha_by_index(index) - # END for each index - # END for each entity - - def size(self): - """:return: amount of packed objects in this database""" - sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes) #} END interface diff --git a/db/ref.py b/db/ref.py index 2c63884bc..5db8d7a23 100644 --- a/db/ref.py +++ b/db/ref.py @@ -1,7 +1,64 @@ -from base import CompoundDB +from base import ( + CompoundDB, + ) +import os __all__ = ('CompoundDB', ) class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" - + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super(ReferenceDB, self).__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + # END handle dbs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + except OSError: + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = set(db.root_path() for db in self._dbs) + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + self._dbs.append(dbcls(path)) + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super(ReferenceDB, self).update_cache(force) diff --git a/test/db/lib.py b/test/db/lib.py index cf752741b..57f5eefc1 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -3,6 +3,7 @@ with_rw_directory, with_packs_rw, ZippedStoreShaWriter, + fixture_path, TestBase ) diff --git a/test/db/test_git.py b/test/db/test_git.py new file mode 100644 index 000000000..35a0d5bad --- /dev/null +++ b/test/db/test_git.py @@ -0,0 +1,9 @@ +from lib import * +from gitdb.db import GitDB + +class TestGitDB(TestBase): + + def test_reading(self): + ldb = GitDB(fixture_path('../../.git/objects') + self.fail("todo") + diff --git a/test/db/test_loose.py b/test/db/test_loose.py index 70cd7742c..536b02048 100644 --- a/test/db/test_loose.py +++ b/test/db/test_loose.py @@ -11,3 +11,8 @@ def test_writing(self, path): self._assert_object_writing(ldb) self._assert_object_writing_async(ldb) + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 89f73f036..f347f408b 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -14,22 +14,22 @@ def test_writing(self, path): # on demand, we init our pack cache num_packs = 2 - assert len(pdb._entities) == num_packs + assert len(pdb.entities()) == num_packs assert pdb._st_mtime != 0 # test pack directory changed: # packs removed - rename a file, should affect the glob - pack_path = pdb._entities[0][1].pack().path() + pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" os.rename(pack_path, new_pack_path) - pdb.update_pack_entity_cache(force=True) - assert len(pdb._entities) == num_packs - 1 + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 # packs added os.rename(new_pack_path, pack_path) - pdb.update_pack_entity_cache(force=True) - assert len(pdb._entities) == num_packs + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs # bang on the cache # access the Entities directly, as there is no iteration interface diff --git a/test/db/test_ref.py b/test/db/test_ref.py new file mode 100644 index 000000000..3b027d701 --- /dev/null +++ b/test/db/test_ref.py @@ -0,0 +1,21 @@ +from lib import * +from gitdb.db import ReferenceDB + +class TestReferenceDB(TestBase): + + @with_rw_directory + def test_writing(self, path): + # TODO: setup alternate file + alternates = + ldb = ReferenceDB(path) + + # try empty, non-existing + + # add two, one is invalid + + # remove valid + + # add valid + + self.fail("todo") + diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 37abc1725..8046c1f83 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -3,6 +3,7 @@ TestBigRepoR ) +from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB import sys @@ -39,7 +40,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) + print >> sys.stderr, "PDB: looked up %i sha in %i packs (random=%i) in %f s ( %f shas/s )" % (ns, len(pdb.entities()), rand, elapsed, ns / elapsed) # END for each random mode elapsed_order, elapsed_rand = access_times @@ -82,10 +83,10 @@ def test_pack_random_access(self): for index in xrange(entity.index().size()): try: assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 except UnsupportedOperation: pass # END ignore old indices - count += 1 # END for each index # END for each entity elapsed = time() - st diff --git a/util.py b/util.py index c460969ca..aa6db4088 100644 --- a/util.py +++ b/util.py @@ -57,6 +57,7 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir rename = os.rename dirname = os.path.dirname +basename = os.path.basename join = os.path.join read = os.read write = os.write From 92ca2e4ad606fbec7c934ad9e467a1b51fddcc92 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 20 Jun 2010 19:47:38 +0200 Subject: [PATCH 0034/3719] Implemented gitdb, it should be a fully functional git database with full read support, and the ability to write loose objects --- db/base.py | 36 ++++++++++++++++++++++++++++++------ db/git.py | 36 ++++++++++++++++++++++++++++++------ db/pack.py | 2 +- db/ref.py | 15 ++++++++++++--- test/db/lib.py | 2 +- test/db/test_git.py | 23 ++++++++++++++++++++--- test/db/test_ref.py | 42 +++++++++++++++++++++++++++++++++++++----- 7 files changed, 131 insertions(+), 25 deletions(-) diff --git a/db/base.py b/db/base.py index 91e82fc0b..35c20b7e1 100644 --- a/db/base.py +++ b/db/base.py @@ -1,13 +1,19 @@ """Contains implementations of database retrieveing objects""" from gitdb.util import ( pool, - join + join, + LazyMixin, + to_bin_sha ) +from gitdb.exc import BadObject + from async import ( ChannelThreadTask ) +from itertools import chain + __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -182,22 +188,40 @@ def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() + def _db_query(self, sha): + """:return: database containing the given 20 or 40 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it everytime a database is being queried + sha = to_bin_sha(sha) + for db in self._dbs: + if db.has_object(sha): + return db + # END for each database + raise BadObject(sha) + #{ ObjectDBR interface def has_object(self, sha): - raise NotImplementedError("To be implemented in subclass") + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions def info(self, sha): - raise NotImplementedError("To be implemented in subclass") + return self._db_query(sha).info(sha) def stream(self, sha): - raise NotImplementedError() + return self._db_query(sha).stream(sha) def size(self): - raise NotImplementedError() + """:return: total size of all contained databases""" + return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) def sha_iter(self): - raise NotImplementedError() + return chain(*(db.sha_iter() for db in self._dbs)) #} END object DBR Interface diff --git a/db/git.py b/db/git.py index 1953def17..ad9a613b3 100644 --- a/db/git.py +++ b/db/git.py @@ -1,7 +1,8 @@ from base import ( - CompoundDB, - FileDBBase, - ) + CompoundDB, + ObjectDBW, + FileDBBase + ) from loose import LooseObjectDB from pack import PackedDB @@ -13,7 +14,7 @@ __all__ = ('GitDB', ) -class GitDB(FileDBBase, CompoundDB): +class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" # Configuration @@ -22,7 +23,7 @@ class GitDB(FileDBBase, CompoundDB): ReferenceDBCls = ReferenceDB # Directories - packs_dir = 'packs' + packs_dir = 'pack' loose_dir = '' alternates_dir = os.path.join('info', 'alternates') @@ -31,19 +32,42 @@ def __init__(self, root_path): super(GitDB, self).__init__(root_path) def _set_cache_(self, attr): - if attr == '_dbs': + if attr == '_dbs' or attr == '_loose_db': self._dbs = list() + loose_db = None for subpath, dbcls in ((self.packs_dir, self.PackDBCls), (self.loose_dir, self.LooseDBCls), (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) if os.path.exists(path): self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db # END check path exists # END for each db type # should have at least one subdb if not self._dbs: raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + # END handle dbs + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + #} END objectdbw interface diff --git a/db/pack.py b/db/pack.py index 92fbc616b..af6f7ffd6 100644 --- a/db/pack.py +++ b/db/pack.py @@ -110,7 +110,7 @@ def sha_iter(self): def size(self): sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes) + return reduce(lambda x,y: x+y, sizes, 0) #} END object db read diff --git a/db/ref.py b/db/ref.py index 5db8d7a23..3a4813979 100644 --- a/db/ref.py +++ b/db/ref.py @@ -3,7 +3,7 @@ ) import os -__all__ = ('CompoundDB', ) +__all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" @@ -35,7 +35,7 @@ def _update_dbs_from_ref_file(self): ref_paths = list() try: ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] - except OSError: + except (OSError, IOError): pass # END handle alternates @@ -55,7 +55,16 @@ def _update_dbs_from_ref_file(self): # sort them to maintain order added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) for path in added_paths: - self._dbs.append(dbcls(path)) + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception, e: + # ignore invalid paths or issues + pass # END for each path to add def update_cache(self, force=False): diff --git a/test/db/lib.py b/test/db/lib.py index 57f5eefc1..2c597fbd3 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -22,7 +22,7 @@ from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_git.py b/test/db/test_git.py index 35a0d5bad..4d463f1f7 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,9 +1,26 @@ from lib import * from gitdb.db import GitDB +from gitdb.base import OStream, OInfo -class TestGitDB(TestBase): +class TestGitDB(TestDBBase): def test_reading(self): - ldb = GitDB(fixture_path('../../.git/objects') - self.fail("todo") + gdb = GitDB(fixture_path('../../.git/objects')) + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + assert gdb.size() > 200 + assert len(list(gdb.sha_iter())) == gdb.size() + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) + self._assert_object_writing_async(gdb) diff --git a/test/db/test_ref.py b/test/db/test_ref.py index 3b027d701..68d9b8116 100644 --- a/test/db/test_ref.py +++ b/test/db/test_ref.py @@ -1,21 +1,53 @@ from lib import * from gitdb.db import ReferenceDB -class TestReferenceDB(TestBase): +import os + +class TestReferenceDB(TestDBBase): + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + alt_file = open(alt_path, "wb") + for alt in alt_list: + alt_file.write(alt + "\n") + alt_file.close() @with_rw_directory def test_writing(self, path): - # TODO: setup alternate file - alternates = - ldb = ReferenceDB(path) + null_sha_bin = '\0' * 20 + null_sha_hex = "0" * 40 + + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 # try empty, non-existing + assert not rdb.has_object(null_sha_hex) + assert not rdb.has_object(null_sha_bin) + + # setup alternate file # add two, one is invalid + own_repo_path = fixture_path('../../.git/objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + assert rdb.has_object(gitdb_sha) # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 - self.fail("todo") From 92e6770be0f65393199432dcfd24f3f1b10d015e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 22 Jun 2010 10:58:45 +0200 Subject: [PATCH 0035/3719] Added MemoryDB including initial test, moved ZippedShaWriter into stream module, it was just a test helper previously --- base.py | 10 ++++++ db/__init__.py | 1 + db/mem.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ stream.py | 33 +++++++++++++++++++ test/db/lib.py | 29 +++++++++++++++- test/db/test_mem.py | 10 ++++++ test/lib.py | 27 +++------------ test/test_base.py | 4 +-- 8 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 db/mem.py create mode 100644 test/db/test_mem.py diff --git a/base.py b/base.py index 25968f371..0f4e63176 100644 --- a/base.py +++ b/base.py @@ -41,6 +41,16 @@ def __init__(self, *args): def sha(self): return self[0] + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + @property def type(self): return self[1] diff --git a/db/__init__.py b/db/__init__.py index 05d9b21b3..85a0a6874 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -1,6 +1,7 @@ from base import * from loose import * +from mem import * from pack import * from git import * from ref import * diff --git a/db/mem.py b/db/mem.py new file mode 100644 index 000000000..3bb6a339d --- /dev/null +++ b/db/mem.py @@ -0,0 +1,80 @@ +"""Contains the MemoryDatabase implementation""" +from loose import LooseObjectDB +from base import ( + ObjectDBR, + ObjectDBW + ) + +from gitdb.base import OStream +from gitdb.util import to_bin_sha +from gitdb.exc import ( + BadObject, + UnsupportedOperation + ) +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, + ) + +__all__ = ("MemoryDB", ) + +class MemoryDB(ObjectDBR, ObjectDBW): + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO + + :note: memory is currently not threadsafe, hence the async methods cannot be used + for storing""" + + def __init__(self): + super(MemoryDB, self).__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream) + + return istream + + def store_async(self, reader): + raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") + + def has_object(self, sha): + return to_bin_sha(sha) in self._cache + + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + sha = to_bin_sha(sha) + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError: + raise BadObject(sha) + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.iterkeys() diff --git a/stream.py b/stream.py index 6c388a96c..8b7e981b0 100644 --- a/stream.py +++ b/stream.py @@ -499,6 +499,39 @@ def sha(self, as_hex = False): #} END interface + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=os.SEEK_SET): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() + + class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor diff --git a/test/db/lib.py b/test/db/lib.py index 2c597fbd3..b22f9a2b8 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -20,6 +20,7 @@ from async import IteratorReader from cStringIO import StringIO +from struct import pack __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') @@ -29,9 +30,35 @@ class TestDBBase(TestBase): # data two_lines = "1234\nhello world" - all_data = (two_lines, ) + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in xrange(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), StringIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.sha) + + info = db.info(istream.sha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.sha) + assert isinstance(stream, OStream) + assert stream.sha == info.sha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW :note: requires write access to the database""" diff --git a/test/db/test_mem.py b/test/db/test_mem.py new file mode 100644 index 000000000..9e7c1190e --- /dev/null +++ b/test/db/test_mem.py @@ -0,0 +1,10 @@ +from lib import * +from gitdb.db import MemoryDB + +class TestMemoryDB(TestDBBase): + + def test_writing(self): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) diff --git a/test/lib.py b/test/lib.py index 6b25876d6..78817fea9 100644 --- a/test/lib.py +++ b/test/lib.py @@ -2,7 +2,11 @@ from gitdb import ( OStream, ) -from gitdb.stream import Sha1Writer +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter + ) + from gitdb.util import zlib import sys @@ -140,26 +144,5 @@ def _assert(self): assert self.args assert self.myarg - -class ZippedStoreShaWriter(Sha1Writer): - """Remembers everything someone writes to it""" - __slots__ = ('buf', 'zip') - def __init__(self): - Sha1Writer.__init__(self) - self.buf = StringIO() - self.zip = zlib.compressobj(1) # fastest - - def __getattr__(self, attr): - return getattr(self.buf, attr) - - def write(self, data): - alen = Sha1Writer.write(self, data) - self.buf.write(self.zip.compress(data)) - return alen - - def close(self): - self.buf.write(self.zip.flush()) - - #} END stream utilitiess diff --git a/test/test_base.py b/test/test_base.py index c122ec4b1..8d8bcc944 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -31,12 +31,12 @@ def test_streams(self): # test pack info # provides type_id - pinfo = OPackInfo(0, 1, blob_id, s) + pinfo = OPackInfo(0, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha From 9b53ab02cb44571e6167a125a5296b7c3395563f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 22 Jun 2010 11:45:26 +0200 Subject: [PATCH 0036/3719] MemoryDB: Implemented direct stream copy, allowing to flush memory db content into any other object db for permanent storage --- db/loose.py | 24 ++++++++++++++++----- db/mem.py | 33 ++++++++++++++++++++++++++++- stream.py | 51 ++++++++++++++++++++++++++++++++------------- test/db/test_mem.py | 20 ++++++++++++++++-- test/test_stream.py | 4 ++-- 5 files changed, 107 insertions(+), 25 deletions(-) diff --git a/db/loose.py b/db/loose.py index b95d6f1c9..9bcbd6aac 100644 --- a/db/loose.py +++ b/db/loose.py @@ -13,6 +13,7 @@ from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, + FDStream, Sha1Writer ) @@ -43,6 +44,7 @@ import tempfile import mmap +import sys import os @@ -153,13 +155,20 @@ def store(self, istream): if writer is None: # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - writer = FDCompressedSha1Writer(fd) + + if istream.sha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies # END handle custom writer try: try: if istream.sha is not None: - stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, @@ -175,10 +184,15 @@ def store(self, istream): writer.close() # END assure target stream is closed - sha = istream.sha or writer.sha(as_hex=True) + hexsha = None + if istream.sha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha if tmp_path: - obj_path = self.db_path(self.object_path(sha)) + obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) if not isdir(obj_dir): mkdir(obj_dir) @@ -186,7 +200,7 @@ def store(self, istream): rename(tmp_path, obj_path) # END handle dry_run - istream.sha = sha + istream.sha = hexsha return istream def sha_iter(self): diff --git a/db/mem.py b/db/mem.py index 3bb6a339d..9e3d3972d 100644 --- a/db/mem.py +++ b/db/mem.py @@ -5,7 +5,11 @@ ObjectDBW ) -from gitdb.base import OStream +from gitdb.base import ( + OStream, + IStream, + ) + from gitdb.util import to_bin_sha from gitdb.exc import ( BadObject, @@ -16,6 +20,8 @@ DecompressMemMapReader, ) +from cStringIO import StringIO + __all__ = ("MemoryDB", ) class MemoryDB(ObjectDBR, ObjectDBW): @@ -78,3 +84,28 @@ def size(self): def sha_iter(self): return self._cache.iterkeys() + + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + :note: the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existance + + ostream = self.stream(sha) + # compressed data including header + sio = StringIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/stream.py b/stream.py index 8b7e981b0..57a5a194f 100644 --- a/stream.py +++ b/stream.py @@ -25,21 +25,6 @@ #{ RO Streams -class NullStream(object): - """A stream that does nothing but providing a stream interface. - Use it like /dev/null""" - __slots__ = tuple() - - def read(self, size=0): - return '' - - def close(self): - pass - - def write(self, data): - return len(data) - - class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand @@ -113,6 +98,8 @@ def _parse_header_info(self): return type, size + #{ Interface + @classmethod def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream @@ -125,6 +112,10 @@ def new(self, m, close_on_deletion=False): type, size = inst._parse_header_info() return type, size, inst + def data(self): + """:return: random access compatible data we are working on""" + return self._m + def compressed_bytes_read(self): """:return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )""" @@ -171,6 +162,8 @@ def compressed_bytes_read(self): # from the count already return self._cbr + #} END interface + def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" @@ -567,4 +560,32 @@ def close(self): #} END stream interface +class FDStream(object): + """Simple wrapper around a file descriptor""" + __slots__ = "_fd" + def __init__(self, fd): + self._fd = fd + + def write(self, data): + return write(self._fd, data) + + def close(self): + close(self._fd) + + + +class NullStream(object): + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + #} END W streams diff --git a/test/db/test_mem.py b/test/db/test_mem.py index 9e7c1190e..4a9b7ee12 100644 --- a/test/db/test_mem.py +++ b/test/db/test_mem.py @@ -1,10 +1,26 @@ from lib import * -from gitdb.db import MemoryDB +from gitdb.db import ( + MemoryDB, + LooseObjectDB + ) class TestMemoryDB(TestDBBase): - def test_writing(self): + @with_rw_directory + def test_writing(self, path): mdb = MemoryDB() # write data self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/test/test_stream.py b/test/test_stream.py index 41f2b235a..859920719 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -50,7 +50,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle rest if isinstance(stream, DecompressMemMapReader): - assert len(stream._m) == stream.compressed_bytes_read() + assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type rewind_stream(stream) @@ -60,7 +60,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert rdata == cdata if isinstance(stream, DecompressMemMapReader): - assert len(stream._m) == stream.compressed_bytes_read() + assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type def test_decompress_reader(self): From 9e6e7d7f1f624143ef8e8fc04e55e5ca277f43ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 01:09:18 +0200 Subject: [PATCH 0037/3719] Removed redunant code due to parital reimplementation of a file descriptor stream wrapper --- db/loose.py | 14 +++++++------- stream.py | 29 ++++++++++++++++++++++++----- test/lib.py | 11 ++++++----- test/test_util.py | 10 ++++++++++ util.py | 43 +++++++++++-------------------------------- 5 files changed, 58 insertions(+), 49 deletions(-) diff --git a/db/loose.py b/db/loose.py index 9bcbd6aac..f97b98a2b 100644 --- a/db/loose.py +++ b/db/loose.py @@ -174,15 +174,15 @@ def store(self, istream): write_object(istream.type, istream.size, istream.read, writer.write, chunk_size=self.stream_chunk_size) # END handle direct stream copies - except: + finally: if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - finally: + writer.close() + # END assure target stream is closed + except: if tmp_path: - writer.close() - # END assure target stream is closed + os.remove(tmp_path) + raise + # END assure tmpfile removal on error hexsha = None if istream.sha: diff --git a/stream.py b/stream.py index 57a5a194f..3f5d0373a 100644 --- a/stream.py +++ b/stream.py @@ -560,19 +560,38 @@ def close(self): #} END stream interface + class FDStream(object): - """Simple wrapper around a file descriptor""" - __slots__ = "_fd" + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') def __init__(self, fd): self._fd = fd + self._pos = 0 def write(self, data): - return write(self._fd, data) + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos def close(self): close(self._fd) - - + class NullStream(object): """A stream that does nothing but providing a stream interface. diff --git a/test/lib.py b/test/lib.py index 78817fea9..742aa7f5c 100644 --- a/test/lib.py +++ b/test/lib.py @@ -38,11 +38,12 @@ def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) try: - return func(self, path) - except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) - raise - else: + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + raise + finally: shutil.rmtree(path) # END handle exception # END wrapper diff --git a/test/test_util.py b/test/test_util.py index 2272b53e5..6a389d27c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -88,4 +88,14 @@ def test_lockedfd(self): finally: os.remove(my_file) # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/util.py b/util.py index aa6db4088..9d1a96900 100644 --- a/util.py +++ b/util.py @@ -161,35 +161,6 @@ def _set_cache_(self, attr): in the single attribute.""" pass - -class FDStreamWrapper(object): - """A simple wrapper providing the most basic functions on a file descriptor - with the fileobject interface. Cannot use os.fdopen as the resulting stream - takes ownership""" - __slots__ = ("_fd", '_pos') - def __init__(self, fd): - self._fd = fd - self._pos = 0 - - def write(self, data): - self._pos += len(data) - os.write(self._fd, data) - - def read(self, count=0): - if count == 0: - count = os.path.getsize(self._filepath) - # END handle read everything - - bytes = os.read(self._fd, count) - self._pos += len(bytes) - return bytes - - def fileno(self): - return self._fd - - def tell(self): - return self._pos - class LockedFD(object): """This class facilitates a safe read and write operation to a file on disk. @@ -240,7 +211,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode) + fd = os.open(self._lockfilepath(), lockmode, 0600) if not write: os.close(fd) else: @@ -253,11 +224,19 @@ def open(self, write=False, stream=False): # open actual file if required if self._fd is None: # we could specify exlusive here, as we obtained the lock anyway - self._fd = os.open(self._filepath, os.O_RDONLY | binary) + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + os.remove(self._lockfilepath()) + raise + # END handle lockfile # END open descriptor for reading if stream: - return FDStreamWrapper(self._fd) + # need delayed import + from stream import FDStream + return FDStream(self._fd) else: return self._fd # END handle stream From 09275ad268e5459c8edff2cccafd0a43947cabdb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 11:45:09 +0200 Subject: [PATCH 0038/3719] Index and PackFiles do not obtain a lock anymore before reading the files in question - this won't work on read-only alternate repositories anyway, and shouldn't be necessary considering a pack is immutable --- db/loose.py | 5 +++-- pack.py | 16 ++++++---------- util.py | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/db/loose.py b/db/loose.py index f97b98a2b..c9ea3a038 100644 --- a/db/loose.py +++ b/db/loose.py @@ -23,6 +23,7 @@ ) from gitdb.util import ( + file_contents_ro_filepath, ENOENT, to_hex_sha, hex_to_bin, @@ -100,12 +101,12 @@ def _map_loose_object(self, sha): :raise BadObject: if object could not be located""" db_path = self.db_path(self.object_path(to_hex_sha(sha))) try: - fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) except OSError,e: if e.errno != ENOENT: # try again without noatime try: - fd = os.open(db_path, os.O_RDONLY) + return file_contents_ro_filepath(db_path) except OSError: raise BadObject(to_hex_sha(sha)) # didn't work because of our flag, don't try it again diff --git a/pack.py b/pack.py index d6fe56823..c66d6717e 100644 --- a/pack.py +++ b/pack.py @@ -5,10 +5,9 @@ ) from util import ( zlib, - LockedFD, LazyMixin, unpack_from, - file_contents_ro, + file_contents_ro_filepath, ) from fun import ( @@ -140,10 +139,10 @@ def _set_cache_(self, attr): elif attr == "_packfile_checksum": self._packfile_checksum = self._data[-20:] elif attr == "_data": - lfd = LockedFD(self._indexpath) - fd = lfd.open() - self._data = file_contents_ro(fd) - lfd.rollback() + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._data = file_contents_ro_filepath(self._indexpath) else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -337,10 +336,7 @@ def __init__(self, packpath): def _set_cache_(self, attr): if attr == '_data': - ldb = LockedFD(self._packpath) - fd = ldb.open() - self._data = file_contents_ro(fd) - ldb.rollback() + self._data = file_contents_ro_filepath(self._packpath) # read the header information type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) diff --git a/util.py b/util.py index 9d1a96900..f0bf3e60f 100644 --- a/util.py +++ b/util.py @@ -114,7 +114,24 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): if stream: return cStringIO.StringIO(contents) return contents - + +def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): + """Get the file contents at filepath as fast as possible + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + :note: for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY|flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: From d3a0037dd5a11459985e7dc4b6819f6292f20c13 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 15:28:22 +0200 Subject: [PATCH 0039/3719] Fixed critical issues with incorrect permissions set on files the db has written - it was only rw-- for the user that wrote them, but should be readable by everyone by default --- db/loose.py | 5 +++++ util.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/db/loose.py b/db/loose.py index c9ea3a038..a91f0d9d3 100644 --- a/db/loose.py +++ b/db/loose.py @@ -28,6 +28,7 @@ to_hex_sha, hex_to_bin, exists, + chmod, isdir, mkdir, rename, @@ -199,6 +200,10 @@ def store(self, istream): mkdir(obj_dir) # END handle destination directory rename(tmp_path, obj_path) + + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rrr + chmod(obj_path, 0444) # END handle dry_run istream.sha = hexsha diff --git a/util.py b/util.py index f0bf3e60f..2d9838379 100644 --- a/util.py +++ b/util.py @@ -54,6 +54,7 @@ def unpack_from(fmt, data, offset=0): # os shortcuts exists = os.path.exists mkdir = os.mkdir +chmod = os.chmod isdir = os.path.isdir rename = os.rename dirname = os.path.dirname @@ -291,6 +292,9 @@ def _end_writing(self, successful=True): # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + chmod(self._filepath, 0444) else: # just delete the file so far, we failed os.remove(lockfile) From e3d5ad195d9dfa46af3d931f9769e965e337daf7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 25 Jun 2010 15:29:50 +0200 Subject: [PATCH 0040/3719] CompoundDB: implemented simple dict base first-level cache which really helps to improve performance in real-world applications, which need to quickly determine whether objects are in or out for instance, as it happens during index_to_tree conversion Added separate pack streaming test which shows only a throughput of 250 streams / s in a densely packed pack, and about 3.5 MiB of data throughput. Performance tests show that half the time is spent in collecting the numerous deltas, the other one in decompressing and applying them Fixed broken performance tests --- db/base.py | 14 ++++++++++- db/git.py | 5 ++-- db/pack.py | 2 +- db/ref.py | 4 ++- exc.py | 7 +++++- test/performance/test_pack.py | 6 +++-- test/performance/test_pack_streaming.py | 33 +++++++++++++++++++++++++ test/performance/test_stream.py | 1 + 8 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 test/performance/test_pack_streaming.py diff --git a/db/base.py b/db/base.py index 35c20b7e1..0e81f0364 100644 --- a/db/base.py +++ b/db/base.py @@ -183,10 +183,13 @@ class CompoundDB(ObjectDBR, LazyMixin, CachingDB): Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" - def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super(CompoundDB, self)._set_cache_(attr) def _db_query(self, sha): """:return: database containing the given 20 or 40 byte sha @@ -194,8 +197,15 @@ def _db_query(self, sha): # most databases use binary representations, prevent converting # it everytime a database is being queried sha = to_bin_sha(sha) + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + for db in self._dbs: if db.has_object(sha): + self._db_cache[sha] = db return db # END for each database raise BadObject(sha) @@ -232,6 +242,8 @@ def databases(self): return tuple(self._dbs) def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() stat = False for db in self._dbs: if isinstance(db, CachingDB): diff --git a/db/git.py b/db/git.py index ad9a613b3..a9298df05 100644 --- a/db/git.py +++ b/db/git.py @@ -57,8 +57,9 @@ def _set_cache_(self, attr): # finally set the value self._loose_db = loose_db - - # END handle dbs + else: + super(GitDB, self)._set_cache_(attr) + # END handle attrs #{ ObjectDBW interface diff --git a/db/pack.py b/db/pack.py index af6f7ffd6..a78c4a8a0 100644 --- a/db/pack.py +++ b/db/pack.py @@ -46,7 +46,7 @@ def __init__(self, root_path): def _set_cache_(self, attr): if attr == '_entities': self._entities = list() - self.update_cache() + self.update_cache(force=True) # END handle entities initialization def _sort_entities(self): diff --git a/db/ref.py b/db/ref.py index 3a4813979..c149c03d0 100644 --- a/db/ref.py +++ b/db/ref.py @@ -21,7 +21,9 @@ def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() self._update_dbs_from_ref_file() - # END handle dbs + else: + super(ReferenceDB, self)._set_cache_(attr) + # END handle attrs def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls diff --git a/exc.py b/exc.py index 482726e3b..037ac3855 100644 --- a/exc.py +++ b/exc.py @@ -1,4 +1,5 @@ """Module with common exceptions""" +from util import to_hex_sha class ODBError(Exception): """All errors thrown by the object database""" @@ -7,7 +8,11 @@ class InvalidDBRoot(ODBError): """Thrown if an object database cannot be initialized at the given path""" class BadObject(ODBError): - """The object with the given SHA does not exist""" + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) class BadObjectType(ODBError): """The object had an unsupported type""" diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 8046c1f83..66101a35f 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -72,8 +72,10 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - - print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns + def _disabled_test_correctness(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" for crc in range(2): count = 0 st = time() diff --git a/test/performance/test_pack_streaming.py b/test/performance/test_pack_streaming.py new file mode 100644 index 000000000..4d47cdfcc --- /dev/null +++ b/test/performance/test_pack_streaming.py @@ -0,0 +1,33 @@ +"""Specific test for pack streams only""" +from lib import ( + TestBigRepoR + ) + +from gitdb.db.pack import PackedDB + +import os +import sys +from time import time + +class TestPackStreamingPerformance(TestBigRepoR): + + def test_stream_reading(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 5de463ee2..79fa9bc09 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -1,6 +1,7 @@ """Performance data streaming performance""" from lib import TestBigRepoR from gitdb.db import * +from gitdb.base import * from gitdb.stream import * from gitdb.util import pool from gitdb.typ import str_blob_type From 9e313a4773c97425d5c52be34ee21cbe405ddb84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 25 Jun 2010 16:39:25 +0200 Subject: [PATCH 0041/3719] gitdb now uses 20 byte shas internally only, reducing the need to convert shas around all the time, saving previous function calls, and memory after all --- base.py | 43 ++++++++++++++------------------- db/base.py | 17 ++++++------- db/loose.py | 18 +++++++------- db/mem.py | 6 ++--- db/pack.py | 6 +---- test/db/lib.py | 28 ++++++++++----------- test/db/test_git.py | 3 ++- test/db/test_ref.py | 15 +++++++----- test/performance/test_pack.py | 2 +- test/performance/test_stream.py | 13 ++++++---- test/test_base.py | 12 ++++----- test/test_pack.py | 14 +++++------ util.py | 1 + 13 files changed, 85 insertions(+), 93 deletions(-) diff --git a/base.py b/base.py index 0f4e63176..938a09242 100644 --- a/base.py +++ b/base.py @@ -1,7 +1,6 @@ """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( - to_hex_sha, - to_bin_sha, + bin_to_hex, zlib ) @@ -17,13 +16,13 @@ #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provdiing information - about the sha of the object, the type_string as well as the uncompressed size + """Carries information about an object in an ODB, provding information + about the binary sha of the object, the type_string as well as the uncompressed size in bytes. It can be accessed using tuple notation and using attribute access notation:: - assert dbi[0] == dbi.sha + assert dbi[0] == dbi.binsha assert dbi[1] == dbi.type assert dbi[2] == dbi.size @@ -38,18 +37,14 @@ def __init__(self, *args): #{ Interface @property - def sha(self): + def binsha(self): + """:return: our sha as binary, 20 bytes""" return self[0] - + @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) + return bin_to_hex(self[0]) @property def type(self): @@ -197,16 +192,10 @@ def __init__(self, type, size, stream, sha=None): list.__init__(self, (sha, type, size, stream, None)) #{ Interface - @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) + return bin_to_hex(self[0]) def _error(self): """:return: the error that occurred when processing the stream, or None""" @@ -231,13 +220,13 @@ def read(self, size=-1): #{ interface - def _set_sha(self, sha): - self[0] = sha + def _set_binsha(self, binsha): + self[0] = binsha - def _sha(self): + def _binsha(self): return self[0] - sha = property(_sha, _set_sha) + binsha = property(_binsha, _set_binsha) def _type(self): @@ -280,9 +269,13 @@ def __init__(self, sha, exc): tuple.__init__(self, (sha, exc)) @property - def sha(self): + def binsha(self): return self[0] + @property + def hexsha(self): + return bin_to_hex(self[0]) + @property def error(self): """:return: exception instance explaining the failure""" diff --git a/db/base.py b/db/base.py index 0e81f0364..c687166f7 100644 --- a/db/base.py +++ b/db/base.py @@ -2,8 +2,7 @@ from gitdb.util import ( pool, join, - LazyMixin, - to_bin_sha + LazyMixin ) from gitdb.exc import BadObject @@ -20,8 +19,7 @@ class ObjectDBR(object): """Defines an interface for object database lookup. - Objects are identified either by hex-sha (40 bytes) or - by sha (20 bytes)""" + Objects are identified either by their 20 byte bin sha""" def __contains__(self, sha): return self.has_obj @@ -29,14 +27,14 @@ def __contains__(self, sha): #{ Query Interface def has_object(self, sha): """ - :return: True if the object identified by the given 40 byte hexsha or 20 bytes + :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") def has_object_async(self, reader): """Return a reader yielding information about the membership of objects as identified by shas - :param reader: Reader yielding 20 byte or 40 byte shas. + :param reader: Reader yielding 20 byte shas. :return: async.Reader yielding tuples of (sha, bool) pairs which indicate whether the given sha exists in the database or not""" task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) @@ -44,7 +42,7 @@ def has_object_async(self, reader): def info(self, sha): """ :return: OInfo instance - :param sha: 40 bytes hexsha or 20 bytes binary sha + :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") @@ -57,7 +55,7 @@ def info_async(self, reader): def stream(self, sha): """:return: OStream instance - :param sha: 40 bytes hexsha or 20 bytes binary sha + :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") @@ -192,11 +190,10 @@ def _set_cache_(self, attr): super(CompoundDB, self)._set_cache_(attr) def _db_query(self, sha): - """:return: database containing the given 20 or 40 byte sha + """:return: database containing the given 20 byte sha :raise BadObject:""" # most databases use binary representations, prevent converting # it everytime a database is being queried - sha = to_bin_sha(sha) try: return self._db_cache[sha] except KeyError: diff --git a/db/loose.py b/db/loose.py index a91f0d9d3..7afc5bf1b 100644 --- a/db/loose.py +++ b/db/loose.py @@ -25,8 +25,8 @@ from gitdb.util import ( file_contents_ro_filepath, ENOENT, - to_hex_sha, hex_to_bin, + bin_to_hex, exists, chmod, isdir, @@ -100,7 +100,7 @@ def _map_loose_object(self, sha): """ :return: memory map of that file to allow random read access :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(to_hex_sha(sha))) + db_path = self.db_path(self.object_path(bin_to_hex(sha))) try: return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) except OSError,e: @@ -109,11 +109,11 @@ def _map_loose_object(self, sha): try: return file_contents_ro_filepath(db_path) except OSError: - raise BadObject(to_hex_sha(sha)) + raise BadObject(sha) # didn't work because of our flag, don't try it again self._fd_open_flags = 0 else: - raise BadObject(to_hex_sha(sha)) + raise BadObject(sha) # END handle error # END exception handling try: @@ -144,7 +144,7 @@ def stream(self, sha): def has_object(self, sha): try: - self.readable_db_object_path(to_hex_sha(sha)) + self.readable_db_object_path(bin_to_hex(sha)) return True except BadObject: return False @@ -158,7 +158,7 @@ def store(self, istream): # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - if istream.sha is None: + if istream.binsha is None: writer = FDCompressedSha1Writer(fd) else: writer = FDStream(fd) @@ -167,7 +167,7 @@ def store(self, istream): try: try: - if istream.sha is not None: + if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) @@ -187,7 +187,7 @@ def store(self, istream): # END assure tmpfile removal on error hexsha = None - if istream.sha: + if istream.binsha: hexsha = istream.hexsha else: hexsha = writer.sha(as_hex=True) @@ -206,7 +206,7 @@ def store(self, istream): chmod(obj_path, 0444) # END handle dry_run - istream.sha = hexsha + istream.binsha = hex_to_bin(hexsha) return istream def sha_iter(self): diff --git a/db/mem.py b/db/mem.py index 9e3d3972d..f361ab801 100644 --- a/db/mem.py +++ b/db/mem.py @@ -10,7 +10,6 @@ IStream, ) -from gitdb.util import to_bin_sha from gitdb.exc import ( BadObject, UnsupportedOperation @@ -54,7 +53,7 @@ def store(self, istream): # don't provide a size, the stream is written in object format, hence the # header needs decompression decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) - self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) return istream @@ -62,14 +61,13 @@ def store_async(self, reader): raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") def has_object(self, sha): - return to_bin_sha(sha) in self._cache + return sha in self._cache def info(self, sha): # we always return streams, which are infos as well return self.stream(sha) def stream(self, sha): - sha = to_bin_sha(sha) try: ostream = self._cache[sha] # rewind stream for the next one to read diff --git a/db/pack.py b/db/pack.py index a78c4a8a0..1a1c390ea 100644 --- a/db/pack.py +++ b/db/pack.py @@ -5,10 +5,7 @@ CachingDB ) -from gitdb.util import ( - to_bin_sha, - LazyMixin - ) +from gitdb.util import LazyMixin from gitdb.exc import ( BadObject, @@ -65,7 +62,6 @@ def _pack_info(self, sha): self._sort_entities() # END update sorting - sha = to_bin_sha(sha) for item in self._entities: index = item[2](sha) if index is not None: diff --git a/test/db/lib.py b/test/db/lib.py index b22f9a2b8..0080d919e 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -41,15 +41,15 @@ def _assert_object_writing_simple(self, db): istream = IStream(str_blob_type, len(data), StringIO(data)) new_istream = db.store(istream) assert new_istream is istream - assert db.has_object(istream.sha) + assert db.has_object(istream.binsha) - info = db.info(istream.sha) + info = db.info(istream.binsha) assert isinstance(info, OInfo) assert info.type == istream.type and info.size == istream.size - stream = db.stream(istream.sha) + stream = db.stream(istream.binsha) assert isinstance(stream, OStream) - assert stream.sha == info.sha and stream.type == info.type + assert stream.binsha == info.binsha and stream.type == info.type assert stream.read() == data # END for each item @@ -80,10 +80,10 @@ def _assert_object_writing(self, db): # store returns same istream instance, with new sha set my_istream = db.store(istream) - sha = istream.sha + sha = istream.binsha assert my_istream is istream assert db.has_object(sha) != dry_run - assert len(sha) == 40 # for now we require 40 byte shas as default + assert len(sha) == 20 # verify data - the slow way, we want to run code if not dry_run: @@ -107,12 +107,12 @@ def _assert_object_writing(self, db): # identical to what we fed in ostream.seek(0) istream.stream = ostream - assert istream.sha is not None - prev_sha = istream.sha + assert istream.binsha is not None + prev_sha = istream.binsha db.set_ostream(ZippedStoreShaWriter()) db.store(istream) - assert istream.sha == prev_sha + assert istream.binsha == prev_sha new_ostream = db.ostream() # note: only works as long our store write uses the same compression @@ -143,12 +143,12 @@ def istream_generator(offset=0, ni=ni): for stream in istreams: assert stream.error is None - assert len(stream.sha) == 40 + assert len(stream.binsha) == 20 assert isinstance(stream, IStream) # END assert each stream # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) hasobject_reader = db.has_object_async(reader) count = 0 for sha, has_object in hasobject_reader: @@ -158,7 +158,7 @@ def istream_generator(offset=0, ni=ni): assert count == ni # read the objects we have just written - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) ostream_reader = db.stream_async(reader) # read items individually to prevent hitting possible sys-limits @@ -171,7 +171,7 @@ def istream_generator(offset=0, ni=ni): assert count == ni # get info about our items - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) info_reader = db.info_async(reader) count = 0 @@ -186,7 +186,7 @@ def istream_generator(offset=0, ni=ni): # add 2500 items, and obtain their output streams nni = 2500 reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ] + istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] istream_reader = db.store_async(reader) istream_reader.set_post_cb(istream_to_sha) diff --git a/test/db/test_git.py b/test/db/test_git.py index 4d463f1f7..779e3f15e 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,6 +1,7 @@ from lib import * from gitdb.db import GitDB from gitdb.base import OStream, OInfo +from gitdb.util import hex_to_bin class TestGitDB(TestDBBase): @@ -11,7 +12,7 @@ def test_reading(self): assert 1 < len(gdb.databases()) < 4 # access should be possible - gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) assert gdb.size() > 200 diff --git a/test/db/test_ref.py b/test/db/test_ref.py index 68d9b8116..9df25cef6 100644 --- a/test/db/test_ref.py +++ b/test/db/test_ref.py @@ -1,6 +1,11 @@ from lib import * from gitdb.db import ReferenceDB - + +from gitdb.util import ( + NULL_BIN_SHA, + hex_to_bin + ) + import os class TestReferenceDB(TestDBBase): @@ -15,8 +20,7 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - null_sha_bin = '\0' * 20 - null_sha_hex = "0" * 40 + NULL_BIN_SHA = '\0' * 20 alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) @@ -25,8 +29,7 @@ def test_writing(self, path): assert len(list(rdb.sha_iter())) == 0 # try empty, non-existing - assert not rdb.has_object(null_sha_hex) - assert not rdb.has_object(null_sha_bin) + assert not rdb.has_object(NULL_BIN_SHA) # setup alternate file @@ -37,7 +40,7 @@ def test_writing(self, path): assert len(rdb.databases()) == 1 # we should now find a default revision of ours - gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert rdb.has_object(gitdb_sha) # remove valid diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 66101a35f..af468b0be 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -56,7 +56,7 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) # END for each function # retrieve stream and read all diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 79fa9bc09..1afc1a1a0 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -3,7 +3,10 @@ from gitdb.db import * from gitdb.base import * from gitdb.stream import * -from gitdb.util import pool +from gitdb.util import ( + pool, + bin_to_hex + ) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size @@ -73,10 +76,10 @@ def test_large_data_streaming(self, path): # writing - due to the compression it will seem faster than it is st = time() - sha = ldb.store(IStream('blob', size, stream)).sha + sha = ldb.store(IStream('blob', size, stream)).binsha elapsed_add = time() - st assert ldb.has_object(sha) - db_file = ldb.readable_db_object_path(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) fsize_kib = os.path.getsize(db_file) / 1000 @@ -151,7 +154,7 @@ def istream_iter(): # chunk size is not important as the stream will not really be decompressed # until its read - istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) + istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) ostream_reader = ldb.stream_async(istream_reader) chunk_task = TestStreamReader(ostream_reader, "chunker", None) @@ -172,7 +175,7 @@ def istream_iter(): istream_reader = ldb.store_async(reader) istream_reader.task().max_chunksize = 1 - istream_to_sha = lambda items: [ i.sha for i in items ] + istream_to_sha = lambda items: [ i.binsha for i in items ] istream_reader.set_post_cb(istream_to_sha) ostream_reader = ldb.stream_async(istream_reader) diff --git a/test/test_base.py b/test/test_base.py index 8d8bcc944..740e50bcd 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -7,7 +7,7 @@ from gitdb import * from gitdb.util import ( - NULL_HEX_SHA + NULL_BIN_SHA ) from gitdb.typ import ( @@ -19,12 +19,12 @@ class TestBaseTypes(TestBase): def test_streams(self): # test info - sha = NULL_HEX_SHA + sha = NULL_BIN_SHA s = 20 blob_id = 3 info = OInfo(sha, str_blob_type, s) - assert info.sha == sha + assert info.binsha == sha assert info.type == str_blob_type assert info.type_id == blob_id assert info.size == s @@ -72,9 +72,9 @@ def test_streams(self): # test istream istream = IStream(str_blob_type, s, stream) - assert istream.sha == None - istream.sha = sha - assert istream.sha == sha + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha assert len(istream.binsha) == 20 assert len(istream.hexsha) == 40 diff --git a/test/test_pack.py b/test/test_pack.py index 6821097ad..eaa2d38eb 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -134,8 +134,8 @@ def test_pack_entity(self): count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): count += 1 - assert info.sha == stream.sha - assert len(info.sha) == 20 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 assert info.type_id == stream.type_id assert info.size == stream.size @@ -143,17 +143,17 @@ def test_pack_entity(self): assert not info.type_id in delta_types # try all calls - assert len(entity.collect_streams(info.sha)) - assert isinstance(entity.info(info.sha), OInfo) - assert isinstance(entity.stream(info.sha), OStream) + assert len(entity.collect_streams(info.binsha)) + assert isinstance(entity.info(info.binsha), OInfo) + assert isinstance(entity.stream(info.binsha), OStream) # verify the stream try: - assert entity.is_valid_stream(info.sha, use_crc=True) + assert entity.is_valid_stream(info.binsha, use_crc=True) except UnsupportedOperation: pass # END ignore version issues - assert entity.is_valid_stream(info.sha, use_crc=False) + assert entity.is_valid_stream(info.binsha, use_crc=False) # END for each info, stream tuple assert count == size diff --git a/util.py b/util.py index 2d9838379..4d6f5b1e2 100644 --- a/util.py +++ b/util.py @@ -66,6 +66,7 @@ def unpack_from(fmt, data, offset=0): # constants NULL_HEX_SHA = "0"*40 +NULL_BIN_SHA = "\0"*20 #} END Aliases From c265c97f9130d2225b923b427736796c0a0d957c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 29 Jun 2010 10:56:50 +0200 Subject: [PATCH 0042/3719] Fixed critical bug that could cause single bytes not to be returned in reads that should read all --- stream.py | 4 +--- .../7b/b839852ed5e3a069966281bb08d50012fb309b | Bin 0 -> 446 bytes test/test_stream.py | 15 ++++++++++++--- util.py | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b diff --git a/stream.py b/stream.py index 3f5d0373a..010102bdd 100644 --- a/stream.py +++ b/stream.py @@ -249,9 +249,7 @@ def read(self, size=-1): # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data @@ -269,7 +267,7 @@ def read(self, size=-1): # Note: dcompdat can be empty even though we still appear to have bytes # to read, if we are called by compressed_bytes_read - it manipulates # us to empty the stream - if dcompdat and len(dcompdat) < size and self._br < self._s: + if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: dcompdat += self.read(size-len(dcompdat)) # END handle special case return dcompdat diff --git a/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b new file mode 100644 index 0000000000000000000000000000000000000000..021c2db3456031e1d2ab0bf73701fd07e1c3f18e GIT binary patch literal 446 zcmV;v0YUzF0V^p=O;s>8GGZ_^FfcPQQP4}zEJ-XWDauSLElDkA*k$BmEOsS@@9cup z94Ax`K0BwU`xdG)kwHqqdSOS!%y&8Fc7K0ZetX8c%q|BenU%5@~ z(-?H(E^K=G^sI4tWL0NrNXl|<-X9uJ$(;P;_{5YH25$lN^J%6=Iy|xceG!-L+2w=@ zF(E0*%}-&FY^mnnsdq=;+uk^0*18Z;q%))7P|&5i(pk;rewb_Um*b^Tbx?LaCci_=A6&R?hE*8 ztJ*Xl-{zzfCJ8maG_NQ%C$S_oB_0%@@wthac?_w(_A_#sd|uhiWEPKR`LO@V34K`5 opeZk|%uB|nyow>rIet2~&OwIvJHJh8HJG&J%aR?}0KQhvi8Wi_>i_@% literal 0 HcmV?d00001 diff --git a/test/test_stream.py b/test/test_stream.py index 859920719..dd65a1782 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -4,12 +4,14 @@ DummyStream, Sha1Writer, make_bytes, - make_object + make_object, + fixture_path ) from gitdb import * from gitdb.util import ( - NULL_HEX_SHA + NULL_HEX_SHA, + hex_to_bin ) from gitdb.util import zlib @@ -135,4 +137,11 @@ def test_compressed_writer(self): os.remove(path) # END for each os - + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + diff --git a/util.py b/util.py index 4d6f5b1e2..b55f789c0 100644 --- a/util.py +++ b/util.py @@ -56,6 +56,7 @@ def unpack_from(fmt, data, offset=0): mkdir = os.mkdir chmod = os.chmod isdir = os.path.isdir +isfile = os.path.isfile rename = os.rename dirname = os.path.dirname basename = os.path.basename @@ -288,7 +289,7 @@ def _end_writing(self, successful=True): if self._write and successful: # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": - if os.path.isfile(self._filepath): + if isfile(self._filepath): os.remove(self._filepath) # END remove if exists # END win32 special handling From 155b62a9af0aa7677078331e111d0f7aa6eb4afc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 00:05:38 +0200 Subject: [PATCH 0043/3719] Added empty version of gitdb documentation --- doc/Makefile | 89 ++++++++++++++++++++ doc/source/conf.py | 194 +++++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 20 +++++ 3 files changed, 303 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..b10926ae2 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GitDB.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GitDB.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..58b8791a1 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# GitDB documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 30 00:01:32 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'GitDB' +copyright = u'2010, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0' +# The full version, including alpha/beta/rc tags. +release = '1.0.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'GitDBdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'GitDB.tex', u'GitDB Documentation', + u'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..71ead1fc3 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,20 @@ +.. GitDB documentation master file, created by + sphinx-quickstart on Wed Jun 30 00:01:32 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GitDB's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + From 78e21e74a12f0767f6011a78349af52555ad2f74 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 11:22:05 +0200 Subject: [PATCH 0044/3719] Added auto-doc api reference and fixed plenty of docstrings --- db/base.py | 20 +++++-- db/pack.py | 4 +- doc/.gitignore | 1 + doc/source/api.rst | 113 ++++++++++++++++++++++++++++++++++++++++ doc/source/conf.py | 2 +- doc/source/index.rst | 4 ++ doc/source/intro.rst | 4 ++ doc/source/tutorial.rst | 4 ++ fun.py | 38 +++++++++----- pack.py | 39 +++++++++----- stream.py | 14 +++-- util.py | 18 ++++--- 12 files changed, 218 insertions(+), 43 deletions(-) create mode 100644 doc/.gitignore create mode 100644 doc/source/api.rst create mode 100644 doc/source/intro.rst create mode 100644 doc/source/tutorial.rst diff --git a/db/base.py b/db/base.py index c687166f7..02584c635 100644 --- a/db/base.py +++ b/db/base.py @@ -90,7 +90,9 @@ def __init__(self, *args, **kwargs): #{ Edit Interface def set_ostream(self, stream): - """Adjusts the stream to which all data should be sent when storing new objects + """ + Adjusts the stream to which all data should be sent when storing new objects + :param stream: if not None, the stream to use, if None the default stream will be used. :return: previously installed stream, or None if there was no override @@ -100,13 +102,16 @@ def set_ostream(self, stream): return cstream def ostream(self): - """:return: overridden output stream this instance will write to, or None + """ + :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream def store(self, istream): - """Create a new object in the database + """ + Create a new object in the database :return: the input istream object with its sha set to its corresponding value + :param istream: IStream compatible instance. If its sha is already set to a value, the object will just be stored in the our database format, in which case the input stream is expected to be in object format ( header + contents ). @@ -114,16 +119,19 @@ def store(self, istream): raise NotImplementedError("To be implemented in subclass") def store_async(self, reader): - """Create multiple new objects in the database asynchronously. The method will + """ + Create multiple new objects in the database asynchronously. The method will return right away, returning an output channel which receives the results as they are computed. :return: Channel yielding your IStream which served as input, in any order. The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. + :param reader: async.Reader yielding IStream instances. The same instances will be used in the output channel as were received in by the Reader. + :note:As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" @@ -167,8 +175,10 @@ class CachingDB(object): #{ Interface def update_cache(self, force=False): - """Call this method if the underlying data changed to trigger an update + """ + Call this method if the underlying data changed to trigger an update of the internal caching structures. + :param force: if True, the update must be performed. Otherwise the implementation may decide not to perform an update if it thinks nothing has changed. :return: True if an update was performed as something change indeed""" diff --git a/db/pack.py b/db/pack.py index 1a1c390ea..022efe0a2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -128,8 +128,10 @@ def store_async(self, reader): #{ Interface def update_cache(self, force=False): - """Update our cache with the acutally existing packs on disk. Add new ones, + """ + Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones + :param force: If True, the cache will be updated even though the directory does not appear to have changed according to its modification timestamp. :return: True if the packs have been updated so there is new information, diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..417671da9 --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,113 @@ +.. _api_reference_toplevel: + +############# +API Reference +############# + +**************** +Database.Base +**************** + +.. automodule:: gitdb.db.base + :members: + :undoc-members: + +**************** +Database.Git +**************** + +.. automodule:: gitdb.db.git + :members: + :undoc-members: + +**************** +Database.Loose +**************** + +.. automodule:: gitdb.db.loose + :members: + :undoc-members: + +**************** +Database.Memory +**************** + +.. automodule:: gitdb.db.mem + :members: + :undoc-members: + +**************** +Database.Pack +**************** + +.. automodule:: gitdb.db.pack + :members: + :undoc-members: + +****************** +Database.Reference +****************** + +.. automodule:: gitdb.db.ref + :members: + :undoc-members: + +************ +Base +************ + +.. automodule:: gitdb.base + :members: + :undoc-members: + +************ +Functions +************ + +.. automodule:: gitdb.fun + :members: + :undoc-members: + +************ +Pack +************ + +.. automodule:: gitdb.pack + :members: + :undoc-members: + +************ +Streams +************ + +.. automodule:: gitdb.stream + :members: + :undoc-members: + +************ +Types +************ + +.. automodule:: gitdb.typ + :members: + :undoc-members: + + +************ +Utilities +************ + +.. automodule:: gitdb.util + :members: + :undoc-members: + + + + + + + + + + + diff --git a/doc/source/conf.py b/doc/source/conf.py index 58b8791a1..d8aadabef 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../../')) # -- General configuration ----------------------------------------------------- diff --git a/doc/source/index.rst b/doc/source/index.rst index 71ead1fc3..409c78e99 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,6 +10,10 @@ Contents: .. toctree:: :maxdepth: 2 + + intro + tutorial + api Indices and tables ================== diff --git a/doc/source/intro.rst b/doc/source/intro.rst new file mode 100644 index 000000000..9565e766e --- /dev/null +++ b/doc/source/intro.rst @@ -0,0 +1,4 @@ +######## +Overview +######## + diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 000000000..b23a17703 --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,4 @@ + +######## +Tutorial +######## diff --git a/fun.py b/fun.py index d7e43717c..ccd8c0fc5 100644 --- a/fun.py +++ b/fun.py @@ -39,20 +39,22 @@ # used when dealing with larger streams chunk_size = 1000*mmap.PAGESIZE -__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', - 'write_object' ) +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data' ) #{ Routines def is_loose_object(m): - """:return: True the file contained in memory map m appears to be a loose object. - Only the first two bytes are needed""" + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" b0, b1 = map(ord, m[:2]) word = (b0 << 8) + b1 return b0 == 0x78 and (word % 31) == 0 def loose_object_header_info(m): - """:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well @@ -61,9 +63,10 @@ def loose_object_header_info(m): return type_name, int(size) def pack_object_header_info(data): - """:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) - The type_id should be interpreted according to the ``type_id_to_type_map`` map - The byte-offset specifies the start of the actual zlib compressed datastream + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" c = ord(data[0]) # first byte i = 1 # next char to read @@ -87,8 +90,9 @@ def pack_object_header_info(data): # END handle exceptions def msb_size(data, offset=0): - """:return: tuple(read_bytes, size) read the msb size from the given random - access data starting at the given byte offset""" + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" size = 0 i = 0 l = len(data) @@ -107,12 +111,14 @@ def msb_size(data, offset=0): return i+offset, size def loose_object_header(type, size): - """:return: string representing the loose object header, which is immediately + """ + :return: string representing the loose object header, which is immediately followed by the content stream of size 'size'""" return "%s %i\0" % (type, size) def write_object(type, size, read, write, chunk_size=chunk_size): - """Write the object as identified by type, size and source_stream into the + """ + Write the object as identified by type, size and source_stream into the target_stream :param type: type string of the object @@ -131,8 +137,10 @@ def write_object(type, size, read, write, chunk_size=chunk_size): return tbw def stream_copy(read, write, size, chunk_size): - """Copy a stream up to size bytes using the provided read and write methods, + """ + Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size + :note: its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written @@ -156,8 +164,10 @@ def stream_copy(read, write, size, chunk_size): def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): - """Apply data from a delta buffer using a source buffer to the target file, + """ + Apply data from a delta buffer using a source buffer to the target file, which will be written to + :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes diff --git a/pack.py b/pack.py index c66d6717e..ed873b269 100644 --- a/pack.py +++ b/pack.py @@ -383,8 +383,9 @@ def version(self): return self._version def data(self): - """:return: read-only data of this pack. It provides random access and usually - is a memory map""" + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map""" return self._data def checksum(self): @@ -428,19 +429,22 @@ def collect_streams(self, offset): def info(self, offset): """Retrieve information about the object at the given file-absolute offset + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._data, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._data, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): - """:return: iterator yielding OPackStream compatible instances, allowing - to access the data in the pack directly. + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. :note: Iterating a pack directly is costly as the datastream has to be decompressed @@ -555,6 +559,7 @@ def _object(self, sha, as_stream, index=-1): def info(self, sha): """Retrieve information about the object identified by the given sha + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" @@ -562,6 +567,7 @@ def info(self, sha): def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha + :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" @@ -589,15 +595,18 @@ def index(self): return self._index def is_valid_stream(self, sha, use_crc=False): - """Verify that the stream at the given sha is valid. - :param sha: 20 byte sha1 of the object whose stream to verify + """ + Verify that the stream at the given sha is valid. + :param use_crc: if True, the index' crc for the sha is used to determine - whether the compressed stream of the object is valid. If it is + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the data of the actual undeltified object, as it depends on more than just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -639,18 +648,22 @@ def is_valid_stream(self, sha, use_crc=False): return True def info_iter(self): - """:return: Iterator over all objects in this pack. The iterator yields + """ + :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) def stream_iter(self): - """:return: iterator over all objects in this pack. The iterator yields - OStream instances""" + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" return self._iter_objects(as_stream=True) def collect_streams_at_offset(self, offset): - """As the version in the PackFile, but can resolve REF deltas within this pack + """ + As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) @@ -678,11 +691,13 @@ def collect_streams_at_offset(self, offset): return streams def collect_streams(self, sha): - """As ``PackFile.collect_streams``, but takes a sha instead of an offset. + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. Additionally, ref_delta streams will be resolved within this pack. If this is not possible, the stream will be left alone, hence it is adivsed to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. diff --git a/stream.py b/stream.py index 010102bdd..675ccf27e 100644 --- a/stream.py +++ b/stream.py @@ -77,6 +77,7 @@ def __del__(self): def _parse_header_info(self): """If this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content + :return: parsed type_string, size""" # read header maxb = 512 # should really be enough, cgit uses 8192 I believe @@ -105,6 +106,7 @@ def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. + :param m: memory map on which to oparate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" @@ -117,8 +119,9 @@ def data(self): return self._m def compressed_bytes_read(self): - """:return: number of compressed bytes read. This includes the bytes it - took to decompress the header ( if there was one )""" + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" # ABSTRACT: When decompressing a byte stream, it can be that the first # x bytes which were requested match the first x bytes in the loosely # compressed datastream. This is the worst-case assumption that the reader @@ -406,6 +409,7 @@ def read(self, count=0): def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != os.SEEK_SET: raise ValueError("Can only seek to position 0") @@ -417,11 +421,14 @@ def seek(self, offset, whence=os.SEEK_SET): @classmethod def new(cls, stream_list): - """Convert the given list of streams into a stream which resolves deltas + """ + Convert the given list of streams into a stream which resolves deltas when reading from it. + :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handled""" @@ -526,6 +533,7 @@ def getvalue(self): class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor + :note: operates on raw file descriptors :note: for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") diff --git a/util.py b/util.py index b55f789c0..502ac94dd 100644 --- a/util.py +++ b/util.py @@ -154,25 +154,26 @@ def to_bin_sha(sha): class LazyMixin(object): """ - Base class providing an interface to lazily retrieve attribute values upon + Base class providing an interface to lazily retrieve attribute values upon first access. If slots are used, memory will only be reserved once the attribute - is actually accessed and retrieved the first time. All future accesses will + is actually accessed and retrieved the first time. All future accesses will return the cached value as stored in the Instance's dict or slot. """ + __slots__ = tuple() def __getattr__(self, attr): """ Whenever an attribute is requested that we do not know, we allow it to be created and set. Next time the same attribute is reqeusted, it is simply - returned from our dict/slots. - """ + returned from our dict/slots. """ self._set_cache_(attr) # will raise in case the cache was not created return object.__getattribute__(self, attr) def _set_cache_(self, attr): - """ This method should be overridden in the derived class. + """ + This method should be overridden in the derived class. It should check whether the attribute named by attr can be created and cached. Do nothing if you do not know the attribute or call your subclass @@ -183,7 +184,8 @@ def _set_cache_(self, attr): class LockedFD(object): - """This class facilitates a safe read and write operation to a file on disk. + """ + This class facilitates a safe read and write operation to a file on disk. If we write to 'file', we obtain a lock file at 'file.lock' and write to that instead. If we succeed, the lock file will be renamed to overwrite the original file. @@ -212,7 +214,9 @@ def _lockfilepath(self): return "%s.lock" % self._filepath def open(self, write=False, stream=False): - """Open the file descriptor for reading or writing, both in binary mode. + """ + Open the file descriptor for reading or writing, both in binary mode. + :param write: if True, the file descriptor will be opened for writing. Other wise it will be opened read-only. :param stream: if True, the file descriptor will be wrapped into a simple stream From 4cde3c046b71bf481418cd3a2a0f0bf5bc540a2b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 21:52:12 +0200 Subject: [PATCH 0045/3719] Added a minimal documentation, including a quick usage guide --- doc/source/api.rst | 2 +- doc/source/conf.py | 11 ++-- doc/source/intro.rst | 25 +++++++++ doc/source/tutorial.rst | 116 ++++++++++++++++++++++++++++++++++++++-- test/test_example.py | 53 ++++++++++++++++++ 5 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 test/test_example.py diff --git a/doc/source/api.rst b/doc/source/api.rst index 417671da9..9dea95764 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -1,4 +1,4 @@ -.. _api_reference_toplevel: +.. _api-label: ############# API Reference diff --git a/doc/source/conf.py b/doc/source/conf.py index d8aadabef..8e2585c2f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.0' +version = '0.5' # The full version, including alpha/beta/rc tags. -release = '1.0.0' +release = '0.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -93,10 +93,9 @@ # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} +html_theme_options = { + "stickysidebar": "true" +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 9565e766e..a51d8bff4 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -2,3 +2,28 @@ Overview ######## +The *GitDB* project implements interfaces to allow read and write access to git repositories. In its core lies the *db* package, which contains all database types necessary to read a complete git repository. These are the ``LooseObjectDB``, the ``PackedDB`` and the ``ReferenceDB`` which are combined into the ``GitDB`` to combine every aspect of the git database. + +For this to work, GitDB implements pack reading, as well as loose object reading and writing. Data is always encapsulated in streams, which allows huge files to be handled as well as small ones, usually only chunks of the stream are kept in memory for processing, never the whole stream at once. + +Interfaces are used to describe the API, making it easy to provide alternate implementations. + +================ +Installing GitDB +================ +Its easiest to install gitdb using the *easy_install* program, which is part of the `setuptools`_:: + + $ easy_install gitdb + +As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +=============== +Getting Started +=============== +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index b23a17703..cfe3fb284 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -1,4 +1,114 @@ +.. _tutorial-label: -######## -Tutorial -######## +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying types. + +****** +Design +****** +The *GitDB* project models a standard git object database and implements it in pure python. This means that data, being classified by one of four types, can can be stored in the database and will in future be referred to by the generated SHA1 key, which is a 20 byte string within python. + +*GitDB* implements *RW* access to loose objects, as well as *RO* access to packed objects. Compound Databases allow to combine multiple object databases into one. + +All data is read and written using streams, which effectively prevents more than a chunk of the data being kept in memory at once mostly [#]_. + +******* +Streams +******* +In order to assure the object database can handle objects of any size, a stream interface is used for data retrieval as well as to fill data into the database. + +Basic Stream Types +================== +There are two fundamentally different types of streams, **IStream**\ s and **OStream**\ s. IStreams are mutable and are used to provide data streams to the database to create new objects. + +OStreams are immutable and are used to read data from the database. The base of this type, **OInfo**, contains only type and size information of the queried object, but no stream, which is slightly faster to retrieve depending on the database. + +OStreams are tuples, IStreams are lists. Both, OInfo and OStream, have the same member ordering which allows quick conversion from one type to another. + +**************************** +Data Query and Data Addition +**************************** +Databases support query and/or addition of objects using simple interfaces. They are called **ObjectDBR** for read-only access, and **ObjectDBW** for write access to create new objects. + +Both have two sets of methods, one of which allows interacting with single objects, the other one allowing to handle a stream of objects simultaneously and asynchronously. + +Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: + + + ldb = LooseObjectDB(fixture_path("../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + # END for each sha in database + +To store information, you prepare an *IStream* object with the required information. The provided stream will be read and converted into an object, and the respective 20 byte SHA1 identifier is stored in the IStream object:: + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + +********************** +Asynchronous Operation +********************** +For each read or write method that allows a single-object to be handled, an *_async* version exists which reads items to be processed from a channel, and writes the operation's result into an output channel that is read by the caller or by other async methods, to support chaining. + +Using asynchronous operations is easy, but chaining multiple operations together to form a complex one would require you to read the docs of the *async* package. At the current time, due to the *GIL*, the *GitDB* can only achieve true concurrency during zlib compression and decompression if big objects, if the respective c modules where compiled in *async*. + +Asynchronous operations are scheduled by a *ThreadPool* which resides in the *gitdb.util* module:: + + from gitdb.util import pool + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) + + +Use async methods with readers, which supply items to be processed. The result is given through readers as well:: + + from async import IteratorReader + + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + + +********* +Databases +********* +A database implements different interfaces, one if which will always be the *ObjectDBR* interface to support reading of object information and streams. + +The *Loose Object Database* as well as the *Packed Object Database* are *File Databases*, hence they operate on a directory which contains files they can read. + +File databases implementing the *ObjectDBW* interface can also be forced to write their output into the specified stream, using the ``set_ostream`` method. This effectively allows you to redirect its output to anywhere you like. + +*Compound Databases* are not implementing their own access type, but instead combine multiple database implementations into one. Examples for this database type are the *Reference Database*, which reads object locations from a file, and the *GitDB* which combines loose, packed and referenced objects into one database interface. + +For more information about the individual database types, please see the :ref:`API Reference `, and the unittests for the respective types. + + +---- + +.. [#] When reading streams from packs, all deltas are currently applied and the result written into a memory map before the first byte is returned. Future versions of the delta-apply algorithm might improve on this. diff --git a/test/test_example.py b/test/test_example.py new file mode 100644 index 000000000..dc3d6230a --- /dev/null +++ b/test/test_example.py @@ -0,0 +1,53 @@ +"""Module with examples from the tutorial section of the docs""" +from lib import * +from gitdb import IStream +from gitdb.db import LooseObjectDB +from gitdb.util import pool + +from cStringIO import StringIO + +from async import IteratorReader + +class TestExamples(TestBase): + + def test_base(self): + ldb = LooseObjectDB(fixture_path("../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + + + # async operation + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) From 7562fdd96ab995f6c25fc102ef40a285283c844e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jul 2010 13:30:14 +0200 Subject: [PATCH 0046/3719] added setup.py and Manifest to allow distribution and easy_install --- .gitignore | 3 +++ MANIFEST.in | 15 +++++++++++++++ README | 2 +- doc/source/intro.rst | 12 ++++++++++++ ext/async | 2 +- setup.py | 18 ++++++++++++++++++ 6 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 MANIFEST.in create mode 100755 setup.py diff --git a/.gitignore b/.gitignore index 1a9d961a7..1e097f6f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +MANIFEST +build/ +dist/ *.pyc *.o *.so diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..a01acc452 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +include VERSION +include LICENSE +include CHANGES +include AUTHORS +include README + +include _fun.c + +graft test + +global-exclude .git* +global-exclude *.pyc +global-exclude *.so +global-exclude *.dll +global-exclude *.o diff --git a/README b/README index a52d9f508..33a566d86 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -GtDB +GitDB ===== GitDB allows you to access bare git repositories for reading and writing. It diff --git a/doc/source/intro.rst b/doc/source/intro.rst index a51d8bff4..4d675cbc4 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -26,4 +26,16 @@ Getting Started =============== It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. +================= +Source Repository +================= +The latest source can be cloned using git from one of the following locations: + + * git://gitorious.org/git-python/gitdb.git + * git://github.com/Byron/gitdb.git + +License Information +=================== +*GitDB* is licensed under the New BSD License. + .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/ext/async b/ext/async index 796b5e94f..a18235276 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 796b5e94f19dfc36a3fb251468192373c76510b0 +Subproject commit a1823527631ffa8a2438c95096e71a741df7b61e diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..b6c2c414b --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +from distutils.core import setup, Extension + +setup(name = "gitdb", + version = "0.5.0", + description = "Git Object Database", + author = "Sebastian Thiel", + author_email = "byronimo@gmail.com", + url = "http://gitorious.org/git-python/gitdb", + packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), + package_data={'gitdb' : ['AUTHORS', 'README'], + 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + package_dir = {'gitdb':''}, + ext_modules=[Extension('gitdb._fun', ['_fun.c'])], + license = "BSD License", + requires=('async (>=0.6.0)',), + long_description = """GitDB is a pure-Python git object database""" + ) From b76bac22bbb146a7a82318721147d7a5f831b7e9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jul 2010 18:23:53 +0200 Subject: [PATCH 0047/3719] Removed obsolete makefile, to compile the extension, use ./setup build_ext and copy _fun.so from the build directory into the root directory --- makefile | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 makefile diff --git a/makefile b/makefile deleted file mode 100644 index 390289625..000000000 --- a/makefile +++ /dev/null @@ -1,12 +0,0 @@ - -_fun.o: _fun.c - gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@ - -_fun.so: _fun.o - gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@ - -all: _fun.so - -clean: - -rm *.so - -rm *.o From 6c8721a7d5d32e54bb4ffd3725ed23ac5d76a593 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 2 Jul 2010 16:22:40 +0200 Subject: [PATCH 0048/3719] Win32 compatability adjustments. One performance test fails during shutdown as python fails to delete a stream in time it seems, so the file cannot be deleted as it is still opened by someone. This is madness, raising the question how badly python truly fails to call the destructors of objects in order to allow them to release their resources --- db/loose.py | 16 ++++++++++++++-- test/lib.py | 6 ++++++ test/test_example.py | 3 +++ test/test_stream.py | 5 ++++- util.py | 8 ++++++-- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/db/loose.py b/db/loose.py index 7afc5bf1b..86eb151fb 100644 --- a/db/loose.py +++ b/db/loose.py @@ -30,6 +30,8 @@ exists, chmod, isdir, + isfile, + remove, mkdir, rename, dirname, @@ -60,6 +62,12 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): # chunks in which data will be copied between streams stream_chunk_size = chunk_size + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = 0444 + if os.name == 'nt': + new_objects_mode = 0644 + def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) @@ -199,11 +207,15 @@ def store(self, istream): if not isdir(obj_dir): mkdir(obj_dir) # END handle destination directory + # rename onto existing doesn't work on windows + if os.name == 'nt' and isfile(obj_path): + remove(obj_path) + # END handle win322 rename(tmp_path, obj_path) # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rrr - chmod(obj_path, 0444) + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) # END handle dry_run istream.binsha = hex_to_bin(hexsha) diff --git a/test/lib.py b/test/lib.py index 742aa7f5c..3fb87d547 100644 --- a/test/lib.py +++ b/test/lib.py @@ -19,6 +19,7 @@ import tempfile import shutil import os +import gc #{ Bases @@ -44,6 +45,11 @@ def wrapper(self): print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) raise finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + gc.collect() shutil.rmtree(path) # END handle exception # END wrapper diff --git a/test/test_example.py b/test/test_example.py index dc3d6230a..dc82436ec 100644 --- a/test/test_example.py +++ b/test/test_example.py @@ -21,6 +21,9 @@ def test_base(self): assert len(ostream.read()) == ostream.size assert ldb.has_object(oinfo.binsha) # END for each sha in database + # assure we close all files + del(ostream) + del(oinfo) data = "my data" istream = IStream("blob", len(data), StringIO(data)) diff --git a/test/test_stream.py b/test/test_stream.py index dd65a1782..948cbe766 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -19,6 +19,7 @@ str_blob_type ) +import time import tempfile import os @@ -125,12 +126,14 @@ def test_compressed_writer(self): # for now, just a single write, code doesn't care about chunking assert len(data) == ostream.write(data) ostream.close() + # its closed already self.failUnlessRaises(OSError, os.close, fd) # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY) + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) os.close(fd) assert written_data == zlib.compress(data, 1) # best speed diff --git a/util.py b/util.py index 502ac94dd..6b2f91073 100644 --- a/util.py +++ b/util.py @@ -58,12 +58,14 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir isfile = os.path.isfile rename = os.rename +remove = os.remove dirname = os.path.dirname basename = os.path.basename join = os.path.join read = os.read write = os.write close = os.close +fsync = os.fsync # constants NULL_HEX_SHA = "0"*40 @@ -128,7 +130,7 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): :note: for now we don't try to use O_NOATIME directly as the right value needs to be shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|flags) + fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: return file_contents_ro(fd, stream, allow_mmap) finally: @@ -300,7 +302,9 @@ def _end_writing(self, successful=True): os.rename(lockfile, self._filepath) # assure others can at least read the file - the tmpfile left it at rw-- - chmod(self._filepath, 0444) + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, 0644) else: # just delete the file so far, we failed os.remove(lockfile) From 46bf4710e0f7184ac4875e8037de30b5081bfda2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jul 2010 00:34:41 +0200 Subject: [PATCH 0049/3719] PackEntity: fixed capital bug which would cause a None to be in place of the bin sha when querying infos or streams through an entity --- ext/async | 2 +- pack.py | 4 +++- test/test_pack.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/async b/ext/async index a18235276..76f15fc4b 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit a1823527631ffa8a2438c95096e71a741df7b61e +Subproject commit 76f15fc4b3e3ccb0160d6c887181f29095d16f23 diff --git a/pack.py b/pack.py index ed873b269..5b13bcc56 100644 --- a/pack.py +++ b/pack.py @@ -516,6 +516,9 @@ def _object(self, sha, as_stream, index=-1): # its a little bit redundant here, but it needs to be efficient if index < 0: index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) offset = self._index.offset(index) type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) if as_stream: @@ -551,7 +554,6 @@ def _object(self, sha, as_stream, index=-1): # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) # END handle stream diff --git a/test/test_pack.py b/test/test_pack.py index eaa2d38eb..445ed4a17 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -144,8 +144,12 @@ def test_pack_entity(self): # try all calls assert len(entity.collect_streams(info.binsha)) - assert isinstance(entity.info(info.binsha), OInfo) - assert isinstance(entity.stream(info.binsha), OStream) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None # verify the stream try: From e750ce0be7d3cbf694c095f6519e2e34b2c3332c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 10:40:35 +0200 Subject: [PATCH 0050/3719] Added method to obtain a full sha from a partial sha, for each of the databases. The IndexFile has a new method to retrieve an index from a partial sha, instead of from a full sha, to provide the required functionality --- db/git.py | 65 ++++++++++++++++++++++++++++++++++++++++++- db/loose.py | 20 ++++++++++++- db/pack.py | 22 +++++++++++++++ exc.py | 6 +++- pack.py | 48 +++++++++++++++++++++++++++++++- test/db/test_git.py | 13 +++++++-- test/db/test_loose.py | 14 +++++++++- test/db/test_pack.py | 24 ++++++++++++++++ test/test_pack.py | 6 ++++ 9 files changed, 211 insertions(+), 7 deletions(-) diff --git a/db/git.py b/db/git.py index a9298df05..62fed7fb6 100644 --- a/db/git.py +++ b/db/git.py @@ -9,11 +9,33 @@ from ref import ReferenceDB from gitdb.util import LazyMixin -from gitdb.exc import InvalidDBRoot +from gitdb.exc import ( + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) import os +from gitdb.util import hex_to_bin + __all__ = ('GitDB', ) + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + + class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" @@ -71,4 +93,45 @@ def ostream(self): def set_ostream(self, ostream): return self._loose_db.set_ostream(ostream) + #} END objectdbw interface + + #{ Interface + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + if len(partial_hexsha) % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if isinstance(db, LooseObjectDB): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface diff --git a/db/loose.py b/db/loose.py index 86eb151fb..521be44c2 100644 --- a/db/loose.py +++ b/db/loose.py @@ -7,7 +7,8 @@ from gitdb.exc import ( InvalidDBRoot, - BadObject, + BadObject, + AmbiguousObjectName ) from gitdb.stream import ( @@ -102,6 +103,23 @@ def readable_db_object_path(self, hexsha): # END handle cache raise BadObject(hexsha) + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(partial_hexsha): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + #} END interface def _map_loose_object(self, sha): diff --git a/db/pack.py b/db/pack.py index 022efe0a2..47d511378 100644 --- a/db/pack.py +++ b/db/pack.py @@ -10,6 +10,7 @@ from gitdb.exc import ( BadObject, UnsupportedOperation, + AmbiguousObjectName ) from gitdb.pack import PackEntity @@ -175,5 +176,26 @@ def update_cache(self, force=False): def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] + + def partial_to_complete_sha(self, partial_binsha): + """:return: 20 byte sha as inferred by the given partial binary sha + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) #} END interface diff --git a/exc.py b/exc.py index 037ac3855..012cdbc6b 100644 --- a/exc.py +++ b/exc.py @@ -13,7 +13,11 @@ class BadObject(ODBError): def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) - + +class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" + class BadObjectType(ODBError): """The object had an unsupported type""" diff --git a/pack.py b/pack.py index 5b13bcc56..ab7c9b677 100644 --- a/pack.py +++ b/pack.py @@ -302,7 +302,53 @@ def sha_to_index(self, sha): # END handle midpoint # END bisect return None - + + def partial_sha_to_index(self, partial_sha): + """:return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_sha: an at least two bytes of a partial sha + :raise AmbiguousObjectName:""" + if len(partial_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + first_byte = ord(partial_sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + len_partial = len(partial_sha) + # fill the partial to full 20 bytes + filled_sha = partial_sha + '\0'*(20 - len_partial) + + # find lowest + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(filled_sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + if lo < self.size: + cur_sha = get_sha(lo) + if cur_sha[:len_partial] == partial_sha: + next_sha = None + if lo+1 < self.size: + next_sha = get_sha(lo+1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_sha) + return lo + # END if we have a match + # END if we found something + return None + if 'PackIndexFile_sha_to_index' in globals(): # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses diff --git a/test/db/test_git.py b/test/db/test_git.py index 779e3f15e..1c0c39c96 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,7 +1,8 @@ from lib import * +from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo -from gitdb.util import hex_to_bin +from gitdb.util import hex_to_bin, bin_to_hex class TestGitDB(TestDBBase): @@ -16,7 +17,15 @@ def test_reading(self): assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) assert gdb.size() > 200 - assert len(list(gdb.sha_iter())) == gdb.size() + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + + # test partial shas + for binsha in sha_list: + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8]) == binsha + # END for each sha + + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") @with_rw_directory def test_writing(self, path): diff --git a/test/db/test_loose.py b/test/db/test_loose.py index 536b02048..8e8a9bfc3 100644 --- a/test/db/test_loose.py +++ b/test/db/test_loose.py @@ -1,10 +1,12 @@ from lib import * from gitdb.db import LooseObjectDB +from gitdb.exc import BadObject +from gitdb.util import bin_to_hex class TestLooseDB(TestDBBase): @with_rw_directory - def test_writing(self, path): + def test_basics(self, path): ldb = LooseObjectDB(path) # write data @@ -16,3 +18,13 @@ def test_writing(self, path): assert shas and len(shas[0]) == 20 assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be foudn + diff --git a/test/db/test_pack.py b/test/db/test_pack.py index f347f408b..c8974b29b 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -2,6 +2,8 @@ from gitdb.db import PackedDB from gitdb.test.lib import fixture_path +from gitdb.exc import BadObject, AmbiguousObjectName + import os import random @@ -44,3 +46,25 @@ def test_writing(self, path): info = pdb.info(sha) stream = pdb.stream(sha) # END for each sha to query + + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambigious ... + # assert num_ambiguous + + # non-existing + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0") diff --git a/test/test_pack.py b/test/test_pack.py index 445ed4a17..a9db19178 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -57,7 +57,13 @@ def _assert_index_file(self, index, version, size): assert entry[0] == index.offset(oidx) assert entry[1] == sha assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4,8,11,17,20): + assert index.partial_sha_to_index(sha[:l]) == oidx + # END for each object index in indexfile + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0") def _assert_pack_file(self, pack, version, size): From e9e0496ea982f7574c6ea379e9f6e85dd111fc8e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 11:10:25 +0200 Subject: [PATCH 0051/3719] Moved partial_to_complete_sha_hex from gitdb to compounddb, its fitting there better --- db/base.py | 61 +++++++++++++++++++++++++++++++++++++++++++-- db/git.py | 56 ----------------------------------------- test/db/test_git.py | 11 ++++++-- 3 files changed, 68 insertions(+), 60 deletions(-) diff --git a/db/base.py b/db/base.py index 02584c635..ebd5040a5 100644 --- a/db/base.py +++ b/db/base.py @@ -2,10 +2,14 @@ from gitdb.util import ( pool, join, - LazyMixin + LazyMixin, + hex_to_bin ) -from gitdb.exc import BadObject +from gitdb.exc import ( + BadObject, + AmbiguousObjectName + ) from async import ( ChannelThreadTask @@ -186,6 +190,22 @@ def update_cache(self, force=False): # END interface + + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): """A database which delegates calls to sub-databases. @@ -258,6 +278,43 @@ def update_cache(self, force=False): # END if is caching db # END for each database to update return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + if len(partial_hexsha) % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + #} END interface diff --git a/db/git.py b/db/git.py index 62fed7fb6..f0e63b15a 100644 --- a/db/git.py +++ b/db/git.py @@ -16,26 +16,9 @@ ) import os -from gitdb.util import hex_to_bin - __all__ = ('GitDB', ) -def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed - and compound databases.""" - if isinstance(database, CompoundDB): - compounds = list() - dbs = database.databases() - output.extend(db for db in dbs if not isinstance(db, CompoundDB)) - for cdb in (db for db in dbs if isinstance(db, CompoundDB)): - _databases_recursive(cdb, output) - else: - output.append(database) - # END handle database type - - - class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" @@ -96,42 +79,3 @@ def set_ostream(self, ostream): #} END objectdbw interface - #{ Interface - - def partial_to_complete_sha_hex(self, partial_hexsha): - """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha - :param partial_hexsha: hexsha with less than 40 byte - :raise AmbiguousObjectName: """ - databases = list() - _databases_recursive(self, databases) - - if len(partial_hexsha) % 2 != 0: - partial_binsha = hex_to_bin(partial_hexsha + "0") - else: - partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - - candidate = None - for db in databases: - full_bin_sha = None - try: - if isinstance(db, LooseObjectDB): - full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) - else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha) - # END handle database type - except BadObject: - continue - # END ignore bad objects - if full_bin_sha: - if candidate and candidate != full_bin_sha: - raise AmbiguousObjectName(partial_hexsha) - candidate = full_bin_sha - # END handle candidate - # END for each db - if not candidate: - raise BadObject(partial_binsha) - return candidate - - #} END interface diff --git a/test/db/test_git.py b/test/db/test_git.py index 1c0c39c96..d2ae10bad 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -20,9 +20,16 @@ def test_reading(self): sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module # test partial shas - for binsha in sha_list: - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8]) == binsha + # this one as uneven and quite short + assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha # END for each sha self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") From ac7d4757ab4041f5f0f5806934130024b098bb82 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 12:10:49 +0200 Subject: [PATCH 0052/3719] Fixed bug caused by incorrect comparison of 'filled-up' binary shas, as it would compare even the filling, instead of taking the canonical length into account and hence ignore the last 4 bytes if the original hex sha had an odd length --- db/base.py | 5 +++-- db/pack.py | 8 ++++++-- fun.py | 21 ++++++++++++++++++++- pack.py | 28 ++++++++++++++++------------ test/db/test_pack.py | 4 ++-- test/test_pack.py | 4 ++-- 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/db/base.py b/db/base.py index ebd5040a5..1914dbbce 100644 --- a/db/base.py +++ b/db/base.py @@ -287,7 +287,8 @@ def partial_to_complete_sha_hex(self, partial_hexsha): databases = list() _databases_recursive(self, databases) - if len(partial_hexsha) % 2 != 0: + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") else: partial_binsha = hex_to_bin(partial_hexsha) @@ -300,7 +301,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if hasattr(db, 'partial_to_complete_sha_hex'): full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha) + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) # END handle database type except BadObject: continue diff --git a/db/pack.py b/db/pack.py index 47d511378..0ec8a4e3b 100644 --- a/db/pack.py +++ b/db/pack.py @@ -177,13 +177,17 @@ def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - def partial_to_complete_sha(self, partial_binsha): + def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for item in self._entities: - item_index = item[1].index().partial_sha_to_index(partial_binsha) + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) if item_index is not None: sha = item[1].index().sha(item_index) if candidate and candidate != sha: diff --git a/fun.py b/fun.py index ccd8c0fc5..e8bc35531 100644 --- a/fun.py +++ b/fun.py @@ -40,7 +40,8 @@ chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data' ) + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha' ) #{ Routines @@ -223,5 +224,23 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" + +def is_equal_canonical_sha(canonical_length, match, sha1): + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length/2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + #} END routines diff --git a/pack.py b/pack.py index ab7c9b677..6966c295f 100644 --- a/pack.py +++ b/pack.py @@ -12,6 +12,7 @@ from fun import ( pack_object_header_info, + is_equal_canonical_sha, type_id_to_type_map, write_object, stream_copy, @@ -303,24 +304,26 @@ def sha_to_index(self, sha): # END bisect return None - def partial_sha_to_index(self, partial_sha): - """:return: index as in `sha_to_index` or None if the sha was not found in this - index file - :param partial_sha: an at least two bytes of a partial sha + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha + :param canonical_length: lenght of the original hexadecimal representation of the + given partial binary sha :raise AmbiguousObjectName:""" - if len(partial_sha) < 2: + if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - first_byte = ord(partial_sha[0]) + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - len_partial = len(partial_sha) # fill the partial to full 20 bytes - filled_sha = partial_sha + '\0'*(20 - len_partial) + filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) # find lowest while lo < hi: @@ -336,14 +339,15 @@ def partial_sha_to_index(self, partial_sha): lo = mid + 1 # END handle midpoint # END bisect - if lo < self.size: + + if lo < self.size(): cur_sha = get_sha(lo) - if cur_sha[:len_partial] == partial_sha: + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size: + if lo+1 < self.size(): next_sha = get_sha(lo+1) if next_sha and next_sha == cur_sha: - raise AmbiguousObjectName(partial_sha) + raise AmbiguousObjectName(partial_bin_sha) return lo # END if we have a match # END if we found something diff --git a/test/db/test_pack.py b/test/db/test_pack.py index c8974b29b..0386b3f80 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -55,7 +55,7 @@ def test_writing(self, path): for i, sha in enumerate(sha_list): short_sha = sha[:max((i % max_bytes), min_bytes)] try: - assert pdb.partial_to_complete_sha(short_sha) == sha + assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha except AmbiguousObjectName: num_ambiguous += 1 pass # valid, we can have short objects @@ -67,4 +67,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0") + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/test/test_pack.py b/test/test_pack.py index a9db19178..717d07e46 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -60,10 +60,10 @@ def _assert_index_file(self, index, version, size): # verify partial sha for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l]) == oidx + assert index.partial_sha_to_index(sha[:l], l*2) == oidx # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0") + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) def _assert_pack_file(self, pack, version, size): From f534e6e9a24f2ac7e7e0f3679551b512d4af569a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jul 2010 11:24:20 +0200 Subject: [PATCH 0053/3719] Added fixes to setup.py to allow easy_installation --- ext/async | 2 +- setup.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/ext/async b/ext/async index 76f15fc4b..081978422 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 76f15fc4b3e3ccb0160d6c887181f29095d16f23 +Subproject commit 0819784229dc98f92d2c57d740c9aebd533846d6 diff --git a/setup.py b/setup.py index b6c2c414b..1afeb3e17 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,60 @@ #!/usr/bin/env python from distutils.core import setup, Extension - +from distutils.command.build_py import build_py + +import os, sys + +# wow, this is a mixed bag ... I am pretty upset about all of this ... +setuptools_build_py_module = None +try: + # don't pull it in if we don't have to + if 'setuptools' in sys.modules: + import setuptools.command.build_py as setuptools_build_py_module +except ImportError: + pass + +def get_data_files(self): + """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, + the line dealing with the ``plen`` has a bug which causes it to truncate too much. + It is fixed in the system interpreters as they receive patches, and shows how + bad it is if something doesn't have proper unittests. + The code here is a plain copy of the python2.6 version which works for all. + + Generate list of '(package,src_dir,build_dir,filenames)' tuples""" + data = [] + if not self.packages: + return data + + # this one is just for the setup tools ! They don't iniitlialize this variable + # when they should, but do it on demand using this method.Its crazy + if hasattr(self, 'analyze_manifest'): + self.analyze_manifest() + # END handle setuptools ... + + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir)+1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + +build_py.get_data_files = get_data_files +if setuptools_build_py_module: + setuptools_build_py_module.build_py._get_data_files = get_data_files +# END apply setuptools patch too + setup(name = "gitdb", version = "0.5.0", description = "Git Object Database", @@ -14,5 +68,6 @@ ext_modules=[Extension('gitdb._fun', ['_fun.c'])], license = "BSD License", requires=('async (>=0.6.0)',), + install_requires='async >= 0.6.0', long_description = """GitDB is a pure-Python git object database""" ) From ebf1ead5fb593bc559a96581b60f05165967d35d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 9 Jul 2010 11:14:47 +0200 Subject: [PATCH 0054/3719] stream: adjusted code to allow compilation in python 2.4. It doesn't work though as the mmap implementation isn't advanced enough, but parent projects can at least import gitdb, and possibly use different implementations on demand --- ext/async | 2 +- stream.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/async b/ext/async index 081978422..2842d1eae 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 0819784229dc98f92d2c57d740c9aebd533846d6 +Subproject commit 2842d1eaee8411f7d09c6dc533465b2a404740ec diff --git a/stream.py b/stream.py index 675ccf27e..90bfc996d 100644 --- a/stream.py +++ b/stream.py @@ -167,7 +167,7 @@ def compressed_bytes_read(self): #} END interface - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != os.SEEK_SET: @@ -407,7 +407,7 @@ def read(self, count=0): self._br += len(data) return data - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" @@ -517,7 +517,7 @@ def write(self, data): def close(self): self.buf.write(self.zip.flush()) - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" if offset != 0 or whence != os.SEEK_SET: From 18152febd428e67b86bb4fb68ec1691d4de75a9c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 9 Jul 2010 11:18:06 +0200 Subject: [PATCH 0055/3719] Bumped version to 0.5.1, added changelog to documentation --- doc/source/changes.rst | 12 ++++++++++++ doc/source/conf.py | 2 +- doc/source/index.rst | 1 + setup.py | 6 +++--- stream.py | 6 +++--- util.py | 43 ++++++++++++++++++++++++++++++++++++++---- 6 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 doc/source/changes.rst diff --git a/doc/source/changes.rst b/doc/source/changes.rst new file mode 100644 index 000000000..84738ae05 --- /dev/null +++ b/doc/source/changes.rst @@ -0,0 +1,12 @@ +######### +Changelog +######### +***** +0.5.1 +***** +* Restored most basic python 2.4 compatibility, such that gitdb can be imported within python 2.4, pack access cannot work though. This at least allows Super-Projects to provide their own workarounds, or use everything but pack support. + +***** +0.5.0 +***** +Initial Release diff --git a/doc/source/conf.py b/doc/source/conf.py index 8e2585c2f..e10addb8d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.0' +release = '0.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst index 409c78e99..d414cb22f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: intro tutorial api + changes Indices and tables ================== diff --git a/setup.py b/setup.py index 1afeb3e17..73d39d09f 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ def get_data_files(self): # END apply setuptools patch too setup(name = "gitdb", - version = "0.5.0", + version = "0.5.1", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", @@ -67,7 +67,7 @@ def get_data_files(self): package_dir = {'gitdb':''}, ext_modules=[Extension('gitdb._fun', ['_fun.c'])], license = "BSD License", - requires=('async (>=0.6.0)',), - install_requires='async >= 0.6.0', + requires=('async (>=0.6.1)',), + install_requires='async >= 0.6.1', long_description = """GitDB is a pure-Python git object database""" ) diff --git a/stream.py b/stream.py index 90bfc996d..3e55c83a4 100644 --- a/stream.py +++ b/stream.py @@ -170,7 +170,7 @@ def compressed_bytes_read(self): def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset @@ -411,7 +411,7 @@ def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self._br = 0 @@ -520,7 +520,7 @@ def close(self): def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self.buf.seek(0) diff --git a/util.py b/util.py index 6b2f91073..3316c24f7 100644 --- a/util.py +++ b/util.py @@ -3,7 +3,14 @@ import mmap import sys import errno -import cStringIO + +from cStringIO import StringIO + +# in py 2.4, StringIO is only StringI, without write support. +# Hence we must use the python implementation for this +if sys.version_info[1] < 5: + from StringIO import StringIO +# END handle python 2.4 try: import async.mod.zlib as zlib @@ -73,6 +80,29 @@ def unpack_from(fmt, data, offset=0): #} END Aliases +#{ compatibility stuff ... + +class _RandomAccessStringIO(object): + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = StringIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + +#} END compatibility stuff ... #{ Routines @@ -94,7 +124,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return cStringIO.StringIO("\0"*size) + return _RandomAccessStringIO("\0"*size) # END handle memory allocation @@ -109,7 +139,12 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): try: if allow_mmap: # supports stream and random access - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except EnvironmentError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 except OSError: pass # END exception handling @@ -117,7 +152,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: - return cStringIO.StringIO(contents) + return _RandomAccessStringIO(contents) return contents def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): From 449e80bf5f04c9ccc3b1b8926b833227d5fe2d5b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 13 Jul 2010 10:06:27 +0200 Subject: [PATCH 0056/3719] byteswapping in offsets method will only be done if required - previously it was always performed even though the byte order of the system didn't require it --- pack.py | 5 ++++- test/test_example.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pack.py b/pack.py index 6966c295f..91323fcce 100644 --- a/pack.py +++ b/pack.py @@ -52,6 +52,8 @@ from itertools import izip import array import os +import sys + __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -272,7 +274,8 @@ def offsets(self): a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more - a.byteswap() + if sys.byteorder == 'little': + a.byteswap() return a else: return tuple(self.offset(index) for index in xrange(self.size())) diff --git a/test/test_example.py b/test/test_example.py index dc82436ec..3fc3fcf5e 100644 --- a/test/test_example.py +++ b/test/test_example.py @@ -22,9 +22,13 @@ def test_base(self): assert ldb.has_object(oinfo.binsha) # END for each sha in database # assure we close all files - del(ostream) - del(oinfo) - + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + data = "my data" istream = IStream("blob", len(data), StringIO(data)) From 425ecf04aa5038c3d46b01ca20de17c51ef6c4e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 13 Jul 2010 10:52:11 +0200 Subject: [PATCH 0057/3719] Adjusted setup.py to deal more gracefully with build failures of our optional extensions --- setup.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 73d39d09f..265156df2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from distutils.core import setup, Extension from distutils.command.build_py import build_py +from distutils.command.build_ext import build_ext import os, sys @@ -10,9 +11,19 @@ # don't pull it in if we don't have to if 'setuptools' in sys.modules: import setuptools.command.build_py as setuptools_build_py_module + from setuptools.command.build_ext import build_ext except ImportError: pass +class build_ext_nofail(build_ext): + """Doesn't fail when build our optional extensions""" + def run(self): + try: + build_ext.run(self) + except Exception: + print "Ignored failure when building extensions, pure python modules will be used instead" + # END ignore errors + def get_data_files(self): """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, the line dealing with the ``plen`` has a bug which causes it to truncate too much. @@ -55,7 +66,8 @@ def get_data_files(self): setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too -setup(name = "gitdb", +setup(cmdclass={'build_ext':build_ext_nofail}, + name = "gitdb", version = "0.5.1", description = "Git Object Database", author = "Sebastian Thiel", From 274c5c89ba973b0ae7ee56424b160417f33d1d5e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Oct 2010 18:56:53 +0200 Subject: [PATCH 0058/3719] Added frame for actual implementation of the aggregation - for now we just implement a forward-merge of the chunks, then the reverse version once there is some more knowledge on how this works --- ext/async | 2 +- fun.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- stream.py | 26 +++++++++++++++-- util.py | 4 +++ 4 files changed, 109 insertions(+), 6 deletions(-) diff --git a/ext/async b/ext/async index 2842d1eae..5992bb6c8 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 2842d1eaee8411f7d09c6dc533465b2a404740ec +Subproject commit 5992bb6c85973ed81c54c71fef42e2413cd29e88 diff --git a/fun.py b/fun.py index e8bc35531..f18d567ac 100644 --- a/fun.py +++ b/fun.py @@ -41,7 +41,48 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha' ) + 'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas', + 'merge_deltas') + + +#{ Structures + +class DeltaChunk(object): + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data' # chunk of bytes to be added to the target buffer or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data + + #{ Interface + + def abssize(self): + return self.to + self.ts + + def apply(self, source, target): + """Apply own data to the target buffer + :param source: buffer providing source bytes for copy operations + :param target: target buffer large enough to contain all the changes to be applied""" + if self.data is not None: + # APPEND DATA + pass + else: + # COPY DATA FROM SOURCE + pass + # END handle chunk mode + + #} END interface + +#} END structures #{ Routines @@ -164,10 +205,46 @@ def stream_copy(read, write, size, chunk_size): return dbw +def reverse_merge_deltas(dcl, dstreams): + """Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + during the merge process + :param dstreams: iterable of delta stream objects. They must be ordered latest first, + hence the delta to be applied last comes first, then its ancestors + :return: None""" + raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") + +def merge_deltas(dcl, dstreams): + """Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + during the merge process + :param dstreams: iterable of delta stream objects. They must be ordered latest last, + hence the delta to be applied last comes last, its oldest ancestor first + :return: None""" + for ds in dstreams: + buf = ds.read() + i, src_size = msb_size(buf) + i, target_size = msb_size(buf, i) + + # parse the commands + + # END for each delta stream + +def apply_delta_chunks(src_buf, src_buf_size, dcl, target): + """ + Apply data from a delta chunk list and a source buffer to the target stream + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param target: ostream with a write method""" + + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): """ - Apply data from a delta buffer using a source buffer to the target file, - which will be written to + Apply data from a delta buffer using a source buffer to the target file :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes diff --git a/stream.py b/stream.py index 3e55c83a4..74dc2b3c4 100644 --- a/stream.py +++ b/stream.py @@ -7,7 +7,9 @@ from fun import ( msb_size, stream_copy, - apply_delta_data, + apply_delta_data, + apply_delta_chunks, + merge_deltas, delta_types ) @@ -320,9 +322,29 @@ def __init__(self, stream_list): self._br = 0 def _set_cache_(self, attr): + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + dcl = list() + reverse_merge_deltas(dcl, self._dstreams) + + if len(dcl) == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl[-1].abssize() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target) + + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" - # prefetch information buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: diff --git a/util.py b/util.py index 3316c24f7..1ea182025 100644 --- a/util.py +++ b/util.py @@ -117,6 +117,10 @@ def make_sha(source=''): def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessStringIO('') + # END handle empty chunks gracefully + try: return mmap.mmap(-1, size) # read-write by default except EnvironmentError: From afefae6de6fee561842c1a09069f9593b6bd62aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Oct 2010 22:46:51 +0200 Subject: [PATCH 0059/3719] Implemented everything around the actual merge-algorithm, which appears to be the heart of the whole thing --- fun.py | 236 +++++++++++++++++++++++++++++++++++++++++++++++------- stream.py | 14 ++-- 2 files changed, 214 insertions(+), 36 deletions(-) diff --git a/fun.py b/fun.py index f18d567ac..c2cbed504 100644 --- a/fun.py +++ b/fun.py @@ -41,12 +41,42 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas', - 'merge_deltas') + 'is_equal_canonical_sha', 'reverse_merge_deltas', + 'merge_deltas', 'DeltaChunkList') #{ Structures +def _trunc_delta(d, size): + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size""" + if size == 0: + raise ValueError("size to truncate to must not be 0") + if d.ts == size: + return + if size > d.ts: + raise ValueError("Cannot truncate delta 'larger'") + + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE, see _split_delta + +def _move_delta_offset(d, bytes): + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size""" + if bytes >= d.ts: + raise ValueError("Cannot move offset that much") + + d.to += bytes + d.ts -= bytes + if d.data: + d.data = d.data[bytes:] + # END handle data + + class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -68,20 +98,120 @@ def __init__(self, to, ts, so, data): def abssize(self): return self.to + self.ts - def apply(self, source, target): + def apply(self, source, write): """Apply own data to the target buffer :param source: buffer providing source bytes for copy operations - :param target: target buffer large enough to contain all the changes to be applied""" - if self.data is not None: - # APPEND DATA - pass - else: + :param write: write method to call with data to write""" + if self.data is None: # COPY DATA FROM SOURCE - pass + write(buffer(source, self.so, self.ts)) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + if self.ts < len(self.data): + write(self.data[:self.ts]) + else: + write(self.data) + # END handle truncation # END handle chunk mode #} END interface +def _closest_index(dcl, absofs): + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs + :note: global method for performance only, it belongs to DeltaChunkList""" + # TODO: binary search !! + for i,d in enumerate(dcl): + if absofs >= d.to: + return i + # END for each delta absofs + raise AssertionError("Should never be here") + +def _split_delta(dcl, absofs, di=None): + """Split the delta at di into two deltas, adjusting their sizes, absofss and data + accordingly and adding them to the dcl. + :param absofs: absolute absofs at which to split the delta + :param di: a pre-determined delta-index, or None if it should be retrieved + :note: it will not split if it + :return: the closest index which has been split ( usually di if given) + :note: belongs to DeltaChunkList""" + if di is None: + di = _closest_index(dcl, absofs) + + d = dcl[di] + if d.to == absofs or d.abssize() == absofs: + return di + + _trunc_delta(d, absofs - d.to) + + # insert new one + ds = d.abssize() + relsize = absofs - ds + + self.insert(di+1, DeltaChunk( ds, + relsize, + (d.so and ds) or None, + (d.data and d.data[relsize:]) or None)) + # END adjust next one + return di + +def _merge_delta(dcl, d): + """Merge the given DeltaChunk instance into the dcl""" + index = _closest_index(dcl, d.to) + od = dcl[index] + + if d.data is None: + if od.data: + # OVERWRITE DATA + pass + else: + # MERGE SOURCE AREA + pass + # END overwrite data + else: + if od.data: + # MERGE DATA WITH DATA + pass + else: + # INSERT DATA INTO COPY AREA + pass + # END combine or insert data + # END handle chunk mode + + +class DeltaChunkList(list): + """List with special functionality to deal with DeltaChunks""" + + def init(self, size): + """Intialize this instance with chunks defining to fill up size from a base + buffer of equal size""" + if len(self) != 0: + return + # pretend we have one huge delta chunk, which just copies everything + # from source to destination + maxint32 = 2**32 + for x in range(0, size, maxint32): + self.append(DeltaChunk(x, maxint32, x, None)) + # END create copy chunks + offset = x*maxint32 + remainder = size-offset + if remainder: + self.append(DeltaChunk(offset, remainder, offset, None)) + # END handle all done in loop + + def terminate_at(self, size): + """Chops the list at the given size, splitting and removing DeltaNodes + as required""" + di = _closest_index(self, size) + d = self[di] + rsize = size - d.to + if rsize: + _trunc_delta(d, rsize) + # END truncate last node if possible + del(self[di+(rsize!=0):]) + #} END structures #{ Routines @@ -204,12 +334,10 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw - def reverse_merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed - during the merge process + :param dcl: see merge_deltas :param dstreams: iterable of delta stream objects. They must be ordered latest first, hence the delta to be applied last comes first, then its ancestors :return: None""" @@ -218,31 +346,78 @@ def reverse_merge_deltas(dcl, dstreams): def merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + :param dcl: DeltaChunkList, may be empty initially, and will be changed during the merge process :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first :return: None""" for ds in dstreams: - buf = ds.read() - i, src_size = msb_size(buf) - i, target_size = msb_size(buf, i) + db = ds.read() + delta_buf_size = ds.size + + # read header + i, src_size = msb_size(db) + i, target_size = msb_size(db, i) - # parse the commands + if len(dcl) == 0: + dcl.init(target_size) + # END handle empty list + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_size): + break + + _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # TODO: Concatenate multiple deltachunks + _merge_delta(dcl, DeltaChunk(tbw, c, None, db[i:i+c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.terminate_at(target_size) # END for each delta stream -def apply_delta_chunks(src_buf, src_buf_size, dcl, target): - """ - Apply data from a delta chunk list and a source buffer to the target stream - - :param src_buf: random access data from which the delta was created - :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes - :param target: ostream with a write method""" - -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file @@ -250,10 +425,9 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data - :param target_file: file like object to write the result to + :param write: write method taking a chunk of bytes :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 - twrite = target_file.write db = delta_buf while i < delta_buf_size: c = ord(db[i]) @@ -289,9 +463,9 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if (rbound < cp_size or rbound > src_buf_size): break - twrite(buffer(src_buf, cp_off, cp_size)) + write(buffer(src_buf, cp_off, cp_size)) elif c: - twrite(db[i:i+c]) + write(db[i:i+c]) i += c else: raise ValueError("unexpected delta opcode 0") diff --git a/stream.py b/stream.py index 74dc2b3c4..9e507c83b 100644 --- a/stream.py +++ b/stream.py @@ -8,8 +8,8 @@ msb_size, stream_copy, apply_delta_data, - apply_delta_chunks, merge_deltas, + DeltaChunkList, delta_types ) @@ -325,8 +325,8 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = list() - reverse_merge_deltas(dcl, self._dstreams) + dcl = DeltaChunkList() + merge_deltas(dcl, self._dstreams) if len(dcl) == 0: self._size = 0 @@ -338,9 +338,13 @@ def _set_cache_(self, attr): self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target) + # APPLY CHUNKS + write = self._mm_target.write + for dc in dcl: + dc.apply(bbuf, write) + # END for each deltachunk to apply def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" From 4d41a878bf0f9171d9a1a44ef885a58c6a025e0d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 01:07:38 +0200 Subject: [PATCH 0060/3719] Initial version of the merge algorithm - the current implementation will lead to quite some fragementation, but that can be improved on once it works --- fun.py | 131 +++++++++++++++++++++++++++++++++++++++++------------- stream.py | 4 +- 2 files changed, 103 insertions(+), 32 deletions(-) diff --git a/fun.py b/fun.py index c2cbed504..2b1419cbb 100644 --- a/fun.py +++ b/fun.py @@ -47,7 +47,7 @@ #{ Structures -def _trunc_delta(d, size): +def _set_delta_rbound(d, size): """Truncate the given delta to the given size :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size""" @@ -63,7 +63,7 @@ def _trunc_delta(d, size): # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE, see _split_delta -def _move_delta_offset(d, bytes): +def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static :param bytes: amount of bytes to move, must be smaller than delta size""" @@ -71,6 +71,7 @@ def _move_delta_offset(d, bytes): raise ValueError("Cannot move offset that much") d.to += bytes + d.so += bytes d.ts -= bytes if d.data: d.data = d.data[bytes:] @@ -95,7 +96,7 @@ def __init__(self, to, ts, so, data): #{ Interface - def abssize(self): + def rbound(self): return self.to + self.ts def apply(self, source, write): @@ -129,39 +130,35 @@ def _closest_index(dcl, absofs): # END for each delta absofs raise AssertionError("Should never be here") -def _split_delta(dcl, absofs, di=None): - """Split the delta at di into two deltas, adjusting their sizes, absofss and data - accordingly and adding them to the dcl. - :param absofs: absolute absofs at which to split the delta - :param di: a pre-determined delta-index, or None if it should be retrieved - :note: it will not split if it - :return: the closest index which has been split ( usually di if given) +def _split_delta(dcl, d, di, relofs, insert_offset=0): + """Split the delta at di into two deltas, adjusting their sizes, offsets and data + accordingly and adding the new part to the dcl + :param relofs: relative offset at which to split the delta + :param d: delta chunk to split + :param di: index of d in dcl + :param insert_offset: offset for the new split id + :return: newly created DeltaChunk :note: belongs to DeltaChunkList""" - if di is None: - di = _closest_index(dcl, absofs) - - d = dcl[di] - if d.to == absofs or d.abssize() == absofs: - return di + if relofs > d.ts: + raise ValueError("Cannot split behinds a chunks rbound") - _trunc_delta(d, absofs - d.to) + osize = d.ts - relofs + _set_delta_rbound(d, relofs) # insert new one - ds = d.abssize() - relsize = absofs - ds + drb = d.rbound() - self.insert(di+1, DeltaChunk( ds, - relsize, - (d.so and ds) or None, - (d.data and d.data[relsize:]) or None)) - # END adjust next one - return di + nd = DeltaChunk( drb, + osize, + (d.so and d.so + osize) or None, + (d.data and d.data[osize:]) or None ) -def _merge_delta(dcl, d): - """Merge the given DeltaChunk instance into the dcl""" - index = _closest_index(dcl, d.to) - od = dcl[index] + self.insert(di+1+insert_offset, nd) + return nd +def _handle_merge(ld, rd): + """Optimize the layout of the lhs delta and the rhs delta + TODO: Once the default implementation is working""" if d.data is None: if od.data: # OVERWRITE DATA @@ -173,6 +170,7 @@ def _merge_delta(dcl, d): else: if od.data: # MERGE DATA WITH DATA + # overwrite the data at the respective spot pass else: # INSERT DATA INTO COPY AREA @@ -180,6 +178,79 @@ def _merge_delta(dcl, d): # END combine or insert data # END handle chunk mode +def _merge_delta(dcl, d): + """Merge the given DeltaChunk instance into the dcl + :param d: the DeltaChunk to merge""" + cdi = _closest_index(dcl, d.to) # current delta index + cd = dcl[cdi] # current delta + + # either we go at his spot, or after + # cdi either moves one up, or stays + dcl.insert(di + (d.to > cd.to), d) + cdi += d.to == cd.to + + while True: + # are we larger than the current block + if d.to < cd.to: + if d.rbound() >= cd.rbound(): + # xxx|xxx|x + # remove the current item completely + dcl.pop(cdi) + cdi -= 1 + elif d.rbound() > cd.to: + # MOVE ITS LBOUND + # xxx|x--| + _move_delta_lbound(cd, d.rbound() - cd.to) + break + else: + # WE DON'T OVERLAP IT + # this can possibly happen + assert False, "Wow, this can really happen" + break + # END rbound overlap handling + # END lbound overlap handling + else: + if d.to >= cd.rbound(): + #|---|...xx + break + # END + + if d.rbound() >= cd.rbound(): + if d.to == cd.to: + #|xxx|x + # REMOVE CD + dcl.pop(cdi) + cdi -= 1 + else: + # TRUNCATE CD + #|-xx| + _set_delta_rbound(cd, d.to - cd.to) + # END handle offset special case + elif d.to == cd.to: + #|x--| + # we shift it by our size + _move_delta_lbound(cd, d.ts) + else: + #|-x-| + # SPLIT CD AND LBOUND MOVE ITS SECOND PART + # insert offset is required to insert it after us + nd = _split_delta(dcl, cd, cdi, 1) + _move_delta_lbound(nd, d.ts) + break + # END handle rbound overlap + # END handle overlap + + cdi += 1 + if cdi < len(dcl): + cd = dcl[cdi] + else: + break + # END check for end of list + # while our chunk is not completely done + + + + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" @@ -208,7 +279,7 @@ def terminate_at(self, size): d = self[di] rsize = size - d.to if rsize: - _trunc_delta(d, rsize) + _set_delta_rbound(d, rsize) # END truncate last node if possible del(self[di+(rsize!=0):]) diff --git a/stream.py b/stream.py index 9e507c83b..3248ae464 100644 --- a/stream.py +++ b/stream.py @@ -326,7 +326,7 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = DeltaChunkList() - merge_deltas(dcl, self._dstreams) + merge_deltas(dcl, reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -334,7 +334,7 @@ def _set_cache_(self, attr): return # END handle empty list - self._size = dcl[-1].abssize() + self._size = dcl[-1].rbound() self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) From 48fdcf44c536e9ca1008749cf6cd4547303ca519 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 12:02:15 +0200 Subject: [PATCH 0061/3719] Added new pack to test database to get some ascii deltas, which may possibly help visual debugging ( but probably not ) added plenty of debug code, to realize that the copy operations are not yet correct --- fun.py | 120 +++++++++++------- stream.py | 15 ++- ...bf8e71d8c18879e499335762dd95119d93d9f1.idx | Bin 0 -> 2248 bytes ...f8e71d8c18879e499335762dd95119d93d9f1.pack | Bin 0 -> 3732 bytes test/test_pack.py | 7 +- 5 files changed, 93 insertions(+), 49 deletions(-) create mode 100644 test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx create mode 100644 test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack diff --git a/fun.py b/fun.py index 2b1419cbb..2d51f62eb 100644 --- a/fun.py +++ b/fun.py @@ -10,6 +10,7 @@ decompressobj = zlib.decompressobj import mmap +from itertools import islice, izip # INVARIANTS OFS_DELTA = 6 @@ -93,7 +94,10 @@ def __init__(self, to, ts, so, data): self.ts = ts self.so = so self.data = data - + + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + #{ Interface def rbound(self): @@ -105,6 +109,7 @@ def apply(self, source, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE + assert len(source) - self.so - self.ts > 0 write(buffer(source, self.so, self.ts)) else: # APPEND DATA @@ -121,14 +126,16 @@ def apply(self, source, write): def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs + absofs. :note: global method for performance only, it belongs to DeltaChunkList""" # TODO: binary search !! for i,d in enumerate(dcl): - if absofs >= d.to: + if absofs < d.to: + return i-1 + elif absofs == d.to: return i # END for each delta absofs - raise AssertionError("Should never be here") + return len(dcl)-1 def _split_delta(dcl, d, di, relofs, insert_offset=0): """Split the delta at di into two deltas, adjusting their sizes, offsets and data @@ -150,7 +157,7 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): nd = DeltaChunk( drb, osize, - (d.so and d.so + osize) or None, + d.so + osize, (d.data and d.data[osize:]) or None ) self.insert(di+1+insert_offset, nd) @@ -178,45 +185,51 @@ def _handle_merge(ld, rd): # END combine or insert data # END handle chunk mode -def _merge_delta(dcl, d): +def _merge_delta(dcl, dc): """Merge the given DeltaChunk instance into the dcl :param d: the DeltaChunk to merge""" - cdi = _closest_index(dcl, d.to) # current delta index + if len(dcl) == 0: + dcl.append(dc) + return + # END early return on empty list + + cdi = _closest_index(dcl, dc.to) # current delta index cd = dcl[cdi] # current delta # either we go at his spot, or after # cdi either moves one up, or stays - dcl.insert(di + (d.to > cd.to), d) - cdi += d.to == cd.to + #print "insert at %i" % (cdi + (dc.to > cd.to)) + #print cd, dc + dcl.insert(cdi + (dc.to > cd.to), dc) + cdi += dc.to == cd.to while True: # are we larger than the current block - if d.to < cd.to: - if d.rbound() >= cd.rbound(): + if dc.to < cd.to: + if dc.rbound() >= cd.rbound(): # xxx|xxx|x # remove the current item completely dcl.pop(cdi) cdi -= 1 - elif d.rbound() > cd.to: + elif dc.rbound() > cd.to: # MOVE ITS LBOUND # xxx|x--| - _move_delta_lbound(cd, d.rbound() - cd.to) + _move_delta_lbound(cd, dc.rbound() - cd.to) break else: # WE DON'T OVERLAP IT - # this can possibly happen - assert False, "Wow, this can really happen" + # this can actually happen, once multiple streams are merged break # END rbound overlap handling # END lbound overlap handling else: - if d.to >= cd.rbound(): + if dc.to >= cd.rbound(): #|---|...xx break # END - if d.rbound() >= cd.rbound(): - if d.to == cd.to: + if dc.rbound() >= cd.rbound(): + if dc.to == cd.to: #|xxx|x # REMOVE CD dcl.pop(cdi) @@ -224,18 +237,18 @@ def _merge_delta(dcl, d): else: # TRUNCATE CD #|-xx| - _set_delta_rbound(cd, d.to - cd.to) + _set_delta_rbound(cd, dc.to - cd.to) # END handle offset special case - elif d.to == cd.to: + elif dc.to == cd.to: #|x--| # we shift it by our size - _move_delta_lbound(cd, d.ts) + _move_delta_lbound(cd, dc.ts) else: #|-x-| # SPLIT CD AND LBOUND MOVE ITS SECOND PART # insert offset is required to insert it after us nd = _split_delta(dcl, cd, cdi, 1) - _move_delta_lbound(nd, d.ts) + _move_delta_lbound(nd, dc.ts) break # END handle rbound overlap # END handle overlap @@ -248,30 +261,14 @@ def _merge_delta(dcl, d): # END check for end of list # while our chunk is not completely done - + ## DEBUG ## + dcl.check_integrity() class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def init(self, size): - """Intialize this instance with chunks defining to fill up size from a base - buffer of equal size""" - if len(self) != 0: - return - # pretend we have one huge delta chunk, which just copies everything - # from source to destination - maxint32 = 2**32 - for x in range(0, size, maxint32): - self.append(DeltaChunk(x, maxint32, x, None)) - # END create copy chunks - offset = x*maxint32 - remainder = size-offset - if remainder: - self.append(DeltaChunk(offset, remainder, offset, None)) - # END handle all done in loop - def terminate_at(self, size): """Chops the list at the given size, splitting and removing DeltaNodes as required""" @@ -283,6 +280,38 @@ def terminate_at(self, size): # END truncate last node if possible del(self[di+(rsize!=0):]) + ## DEBUG ## + self.check_integrity(size) + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doen't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + if dc.data: + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self)-1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft,rgt in izip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + #} END structures #{ Routines @@ -422,7 +451,8 @@ def merge_deltas(dcl, dstreams): :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first :return: None""" - for ds in dstreams: + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi db = ds.read() delta_buf_size = ds.size @@ -430,10 +460,6 @@ def merge_deltas(dcl, dstreams): i, src_size = msb_size(db) i, target_size = msb_size(db, i) - if len(dcl) == 0: - dcl.init(target_size) - # END handle empty list - # interpret opcodes tbw = 0 # amount of target bytes written while i < delta_buf_size: @@ -475,7 +501,7 @@ def merge_deltas(dcl, dstreams): tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - _merge_delta(dcl, DeltaChunk(tbw, c, None, db[i:i+c])) + _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -487,6 +513,8 @@ def merge_deltas(dcl, dstreams): # END for each delta stream + # print dcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ diff --git a/stream.py b/stream.py index 3248ae464..0cb558d78 100644 --- a/stream.py +++ b/stream.py @@ -346,6 +346,19 @@ def _set_cache_(self, attr): dc.apply(bbuf, write) # END for each deltachunk to apply + self._mm_target.seek(0) + + ## DEBUG ## + mt = self._mm_target + for ds in self._dstreams: + ds.stream.seek(0) + self._bstream.stream.seek(0) + self._set_cache_old(attr) + + import chardet + if chardet.detect(mt[:])['encoding'] == 'ascii': + assert self._mm_target[:] == mt[:] + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" @@ -399,7 +412,7 @@ def _set_cache_old(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf) + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### # finally, swap out source and target buffers. The target is now the diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx new file mode 100644 index 0000000000000000000000000000000000000000..a7d6c7177ef3ca68b3b93adafa738b190db9e7ac GIT binary patch literal 2248 zcmciEdo&Yz902g;F|X~SRHBG!gxeyIOi6J{XiJoEkwg!9v}mk#)KDpOOOoa!k4*Ad znD;C0xLvwl6=OGfg>ht98CUMtIXTB4?l~RZb9c_??ECxve!uT;e|^834l zlFur*v=R&Nt&}L-{Sjhte~~!ED}jW23nYN#GNd85NCx8XfrfM6>nAw7lq{^5l85zD zt6{z7f2IKM{hM{Lw`e`YixeSV&IZW;C?%+|NEzY<8^75E-~TlgsJTEDY+gbQ;^nAA zegS3~8jxGCWf_`~Tcic?Qd?oYls2rFg^tHg&S9KhKc%fF5m)DGnogOgE9cK~C$WMO z={v>5H?FddyT-|R+8ooZDBBr13G1l(`p)Ymb>YL2_#VbKKEM81Z7lAfb>6yClP)cX z-IsDY8|>}twxKz{$YXc+p$a9R@%Bg9mg5Tp4rL`o3VqnD7g*Ed zx{~AUaaP1Wf~#7IlMkEUflgG5BzXJlX8JYSt7Hf%31j_Amx4J&h32mG<5X|^mz-{v z0B59Ma3tS(ZEeW(Yc-E!!vfb&{P`Nxc%yf*cSCyNO)v6^nzI!_*UE zHlJ`}=uB4W$owt6!*Tv3&qS3bDHnu~FH@nsskcqQ6^Yh)9+XEqU`6y#DxaF#AZomY zb}y}-5-aWau}k?PMsCu_N;ASjne>oFj_GZo;teiRSbcX;+Erd-iiUKOk!4JEa1c4s z!LBr^S8l#V;F>2g;BJ>SniX8xCf#Ug7>;DgZ%!F(eS{i$_wv4a&{e)^jB2xX(bwHY zi5&V?f{?C5NQ?EzRn+3yqhb`_8sHyd8T2N7Ssh||WaC{-qpdi17NhQM^7uf$-~0J$ z2gSUa_3c+BN8gJ5Ws_VKaPso7d>UqT&E$6EnqQ`vt0)Yli(9;s@9m>ET%$f4q^6QY zo@0tLJu1VCqC=a$+#Kq#G(Z39f__z*S3r>t@yYu=X&X=Xbu%qrl(a-w>@VBnQt*d^ z9vMY^FJU+v*+!eJ61g1v)%WKMe;v2nJ#tkAY9j*5n1 zim~2UzDj5TgYDjft#h5;xmG4}2iJygVxpjtQHROSp4b_cQ*Y+>>vb1FM%;j4SF6tu z7FR~L8KoABNT>=c>vCIdshyb|Capo-GR2nm$$kQNJD2W3&>yFDTWH(7c4Tj=HJ!8G zJJCPwUFCJ8WvpYD!HtQYpQm*L(1MB!g5fN>U6+P%?>x&-y#3An^8;APnTL4ijRQuE z$WwdiRFOkHADyPoiJAH^oRaO>Vxig8bk7gI?Cr)DTIXk*lc-7k%4_>eTax^wRjchg z^0wjYKPz$8JQ$9>b?k6-OJ$VmrJG+$)i0FOtwzW}RBHArJC;5D&w6AwZlDG4yHVHRv13FtSg280tXZ zK|8`4da4BUaB&A1dmK1&4(fb>z287M)c6~6(32#Xu>TUkTdV$r_n;@D;dh|nJv1NA x9D{pqvSt~!EoI3lZP9k4v=g3(X<=qg4%Q^Gr|y1tEW5xf@2?fh{cJ?LVl%wKv z^pxJP5=oIZLnI0yh!DSQ+o$<{4#sushM+vou1idydElvH&GMXOwm=VaD??kqO4F_>M_4D*c7}Y%lbdGD7|jlqFbS6b4L_o(2mi^lhrBYBPaBMy20l0zJFs?4ruEBO)n+=jL#_VJqdSJ*20b8ViH{ zwuoTfEIXr<+|m*;6O>E8sC)u#e8JiX>3^v>T18;+nFFhyyy=G{@z)2~*u5*n1C{rz zLiJB?Mn@K=_;jWZ@Y$&d@-4nom^MhLwdiE(Dng#Ba8g{Wf}U#A=gw7cn@eWQl*U;a zA%Ad-&Us{bx93$>Kc})Z8PB_(%a$!|Dng+!lM}12LR|a9D9@u>r*j$j3UwJk#f3rP z?At$`R7-?K2lea!V3~tkm0-X?^!ap>$WT|E$n1$9p!*hYnIAYJUC&PClBwGT!*zMW9Kp(3|Mt9OFsRKKv`Ih0`j{gy98aS<2@FF~Oa>wU zS>YZPeCB{^+gIfnY(H1`z5Ez0<&+f~Njlhu4i?lk&Ql!iJEyNMuo@P7S&d6^wid3o zF*5=eHwqLe0}`P`e4Lk;zi9fHhW48{P8(!5gQTeRZC%gth9B5S*#FD&mu7`_HnFd#N=vNyRVPbT?k?jcZ$ShF6;FjVBt;oo*@jNHr{N&U3Ta4{DqBqd|WWei=TbAWvb_RqhpC5 z$6th{GQK6A2cW+)6g<2_A6-+B#_Qpfd(y_)m``LFyGX|4CQPMOis2qLCh^%`HG?p0 z=(NC%g%K9=u~v+u#+GMk^>?-y<8yw?byu;k^XyPg3J$^kNW`>Meu76zt(Mep%FCiL zXn!Lj?vGbC;C-lpSgV;>SYc_QzQOfkHvH!RxjDfr!jnyX15~VfQnH&F{H{I`GS72L zL{NVjH+I})NuKGYq>Ytel~v%gA_>Zo6|@8+OvrGdIp4aC$X}Sjol)@fBYnWB>&Jtv zh-ga5*&i~oMpORF_b`QYjZoCpzyJesT&kOJGpIPPR_p1n9UkWmrkMsr#bGL~H9UB~ z9hAQQU#cT)W8BKN-Srb(7C3=`Ri3NK%v8>}XlzGR1K|-|CC%{sKQ9AMH^788N~O%2(xw3$ybV#!P%A?&tjj z-|a<&+#jm{eucsYjaWAi$=osckQx3@H?$mgHQ2ceUPPwka`7Jo zjsEu)Di}nAj#wwvYTf#^<6yMG_{0Tq^ADBfe^r*1sPbFwC8nG&9;RmhsjPWSVak#T z5u$HYbzj3^K_mZiEql<6g%6CRZbP!Y7wrSZ>%LcMP~tSN;2QWF-5KcyoCQUxINHlN^&xQTdE!~$W6gR&H9e-KMeP`0wquA@r#MJ$AAYAI8iN*sA?ljAkLtlGW1yE;@t zWQPQ@gfOB6{DOpQ-DM^_JWN_rN9?wkQMY{#1E{LMT>`)(=TsL)R$Vh1jk9`n7fMn{ zdrj)>N?Cnn=1O=~?v!#39CNPT0}2POyzoDP*xiuK*k)K8)tCC$t^Fqbx3e;)y5f@e z`rTpz52?P>R{w2tlRZdy>Ntrzmr4FgsO?z`83cy`!Gh~iyRpR*4usq5cW>dSg&Xk2 zvxy$<7t@wbaWS9HydfP@+nW4%OEr6BgX9&H+TCzarFY4s)o}m1#kb))R)GzEo_6dU z&3j8^drWvetlUi|Kfu!&dQf__hirTQXuLKUqzr@a5Rp-O$Ov_1k(Bb9y>MxVu+ZZ9 z_rleD{DKbpn;Ys~vm3_Kw1EK}FJxFZb5@aa{B6AW!!?tI|86r;UW074U0Pb(>@ zD%-GzhF@C`_uhI%9iyA;o~d)3{5Wv!!LWC_)zFRyFnI$iKw?4@B|2J;#wo#$G{=^V zfdde+6Z#*p@(Dy2$F?Z56^)mj7IRNCZWc<)J)H(#5{TM0Dlb`N#o! z+IxfI3uAJn!^*MdP=h>C*kOXP>0An@7jl_WH81*93j@RFIH)93G%?(+D~O^5(@Anyu2U#}#%!3@3g@`J z{6znRRE5yo1#7NTmHC>oro#hM*G)_-hpM%3Zz9=*lboD}I)XCE)M)H2*AponveGP| zi5d6ORxkyz=*4tWPe8F;T1O}o6{qCkopL_1+Dz-82h8OxALCKtAF?N;^u0ZXcyuxy z!B!uC^wicTVdQd+RWX?(CTy+t6+ZGII^fTZJWZmUo4RIXCac~fge!g9OuKE0GbdIF zydz8E@0h=9S9@rCWP=eX9? zlTwG*y=x;ynHRb=J+FE;pQk^c4`gkN%s->; zSyaOO0IQ^1L`dZ0&Jj@xUmG+_Yn7Sok5_)2>i9b~oxyLOg;w8gTg)x-^md;dSAgNR zxF1cFojbJXB(#C=$+4)azmVi_-}MUDn{uU0DRsA@zk~u$jCO?-UGPgg?=M;|%frgw zLp`aMB7r&MgL0`5WRd3RLFHFVpN>?SiSjL|2nzxEY?RCCHs|M?KOpk)LT9oIS_WNo zTA2NLW4qrsSU-YfXJqs}kwfEl=3duThlD1GT=Zjqc8#!*xP=|O0M2$ln){6^V`&*r zi_;j{HT)+0o1BiHd@?TUJk>1QQ>WMwsD*ET9^PnFR)Z&2-X&IfMgCKfuqx?#!#&8Y z;_MK2OY(%+Xh{&OVJf^ja1KMD&Hkc&pI?b=yGwe%-eT4!9mw=-;K`n>V%Zk}Pv-!JYHwmpvG^M0RM2*)HK7<0FK`<6yhXXOe7pov9#s|RS zL8!y}t=Gt}Lod|sRk;^`p(^~0v3rTdxt;_*3_}uq<9PN5sNo{t7#JBT0GxK9`ojY% zqUv3!pF2LwL9Ir9{+JN2?V;5gu>##-h~L#X$Nd4D1aHL2V*msVS~@J3em+63&olOq z{i=O*Xx#9MLNtr48ADwoyRUCN_W^%m_?s(|d)IS~fp!n;(RCM}$Mnz&*P+9N_J1m1 z^J}M@K0|;G4hX+*{DA>#xq(=I)jb2C1t8=9XF#y=(^;CEbGkV(z+dhn-}3h`voZjv z*#d7IK3tDnYgIZEX|rJY%3(AmV$t{?q;Bw0poR>w@JW*2U*UF=XCV^820)RZ--q?Z zW7qF5)XCtFmMSz;45Us<9w|snM3aVyvXcIMg@=+rv>QWvK@Iru^ zYWBuCP_n@OZR5SKTJG6&DoaSb6jwc$n$k0*sL?jqk>nqCR*diSf}f;>B+#P{K6!XJ zYqsoaUk6 vEUvU}!6G)gfHqQi+dFJwW&vClgS>$17F!H9R$XG(1^|! literal 0 HcmV?d00001 diff --git a/test/test_pack.py b/test/test_pack.py index 717d07e46..770a78bad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -34,8 +34,10 @@ class TestPack(TestBase): packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) def _assert_index_file(self, index, version, size): @@ -123,14 +125,15 @@ def test_pack_index(self): def test_pack(self): # there is this special version 3, but apparently its like 2 ... - for packfile, version, size in (self.packfile_v2_1, self.packfile_v2_2): + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): pack = PackFile(packfile) self._assert_pack_file(pack, version, size) # END for each pack to test def test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2)): + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo indexfile, version, size = indexinfo entity = PackEntity(packfile) From 4cf3eac8e9ed6ac2b0bac4a617a08fdf194da4ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 19:01:24 +0200 Subject: [PATCH 0062/3719] initial frame for design change - now chunks can refer to DeltaLists as well, which is required to get the copying right. This surely makes things more coplex, but should still result in a performance improvement --- fun.py | 91 ++++++++++++++++++++++++++++++++++++++++++++----------- stream.py | 7 ++--- 2 files changed, 75 insertions(+), 23 deletions(-) diff --git a/fun.py b/fun.py index 2d51f62eb..a0835746e 100644 --- a/fun.py +++ b/fun.py @@ -86,13 +86,14 @@ class DeltaChunk(object): 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None - 'data' # chunk of bytes to be added to the target buffer or None + 'data' # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts - self.so = so + self.so = sos self.data = data def __repr__(self): @@ -103,11 +104,15 @@ def __repr__(self): def rbound(self): return self.to + self.ts + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is None or not isinstance(self.data, DeltaChunkList) + def apply(self, source, write): """Apply own data to the target buffer :param source: buffer providing source bytes for copy operations :param write: write method to call with data to write""" - if self.data is None: + if self.has_data(): # COPY DATA FROM SOURCE assert len(source) - self.so - self.ts > 0 write(buffer(source, self.so, self.ts)) @@ -166,7 +171,7 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): def _handle_merge(ld, rd): """Optimize the layout of the lhs delta and the rhs delta TODO: Once the default implementation is working""" - if d.data is None: + if d.has_data(): if od.data: # OVERWRITE DATA pass @@ -217,6 +222,7 @@ def _merge_delta(dcl, dc): _move_delta_lbound(cd, dc.rbound() - cd.to) break else: + # xx.|---| # WE DON'T OVERLAP IT # this can actually happen, once multiple streams are merged break @@ -224,7 +230,7 @@ def _merge_delta(dcl, dc): # END lbound overlap handling else: if dc.to >= cd.rbound(): - #|---|...xx + #|---|xx break # END @@ -269,9 +275,30 @@ def _merge_delta(dcl, dc): class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def terminate_at(self, size): + def init(self, size): + """Intialize this instance with chunks defining to fill up size from a base + buffer of equal size + :return: self""" + if len(self) != 0: + return + # pretend we have one huge delta chunk, which just copies everything + # from source to destination + maxint32 = 2**32 + for x in range(0, size, maxint32): + self.append(DeltaChunk(x, maxint32, x, None)) + # END create copy chunks + offset = x*maxint32 + remainder = size-offset + if remainder: + self.append(DeltaChunk(offset, remainder, offset, None)) + # END handle all done in loop + + return self + + def set_rbound(self, size): """Chops the list at the given size, splitting and removing DeltaNodes - as required""" + as required + :return: self""" di = _closest_index(self, size) d = self[di] rsize = size - d.to @@ -283,6 +310,26 @@ def terminate_at(self, size): ## DEBUG ## self.check_integrity(size) + return self + + def connect_with(self, bdlc): + """Connect this instance's delta chunks virtually with the given base. + This means that all copy deltas will simply apply to the given region + of the given base. Afterwards, the base is optimized so that add-deltas + will be truncated to the region actually used, or removed completely where + adequate. This way, memory usage is reduced. + :param bdlc: DeltaChunkList to serve as base""" + raise NotImplementedError("todo") + + def apply(self, bbuf, write): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + raise NotImplementedError("todo") + def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -437,31 +484,31 @@ def stream_copy(read, write, size, chunk_size): def reverse_merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: see merge_deltas + :param dcl: see 3 :param dstreams: iterable of delta stream objects. They must be ordered latest first, hence the delta to be applied last comes first, then its ancestors :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def merge_deltas(dcl, dstreams): +def merge_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: DeltaChunkList, may be empty initially, and will be changed - during the merge process :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first - :return: None""" + :return: DeltaChunkList, containing all operations to apply""" + bdcl = None # data chunk list for initial base + dcl = DeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() delta_buf_size = ds.size # read header - i, src_size = msb_size(db) + i, base_size = msb_size(db) i, target_size = msb_size(db, i) # interpret opcodes - tbw = 0 # amount of target bytes written + tbw = 0 # amount of target bytes written while i < delta_buf_size: c = ord(db[i]) i += 1 @@ -494,14 +541,16 @@ def merge_deltas(dcl, dstreams): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_size): + rbound > base_size): break - _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + # _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) + # _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -509,8 +558,14 @@ def merge_deltas(dcl, dstreams): # END handle command byte # END while processing delta data - dcl.terminate_at(target_size) + # merge the lists ! + if base is not None: + dcl.connect_with(base) + # END handle merge + # prepare next base + base = dcl + dcl = DeltaChunkList() # END for each delta stream # print dcl diff --git a/stream.py b/stream.py index 0cb558d78..efb99d218 100644 --- a/stream.py +++ b/stream.py @@ -325,8 +325,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = DeltaChunkList() - merge_deltas(dcl, reversed(self._dstreams)) + dcl = merge_deltas(reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -342,9 +341,7 @@ def _set_cache_(self, attr): # APPLY CHUNKS write = self._mm_target.write - for dc in dcl: - dc.apply(bbuf, write) - # END for each deltachunk to apply + dcl.apply(bbuf, write) self._mm_target.seek(0) From fbf8f3221ea58de54567e034d8a7b4e23833e14b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 12:35:35 +0200 Subject: [PATCH 0063/3719] Filled in first implementation of all missing methods. Its untested, and currently the lists are truncated physically, which would work, but it would be easier to just remember the changed bounds, and apply it later with these bounds in mind --- fun.py | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 168 insertions(+), 22 deletions(-) diff --git a/fun.py b/fun.py index a0835746e..3e126c3e4 100644 --- a/fun.py +++ b/fun.py @@ -12,6 +12,8 @@ import mmap from itertools import islice, izip +from copy import copy + # INVARIANTS OFS_DELTA = 6 REF_DELTA = 7 @@ -51,33 +53,48 @@ def _set_delta_rbound(d, size): """Truncate the given delta to the given size :param size: size relative to our target offset, may not be 0, must be smaller or equal - to our size""" + to our size + :return: d""" if size == 0: raise ValueError("size to truncate to must not be 0") if d.ts == size: return if size > d.ts: - raise ValueError("Cannot truncate delta 'larger'") + raise ValueError("Cannot extend rbound") d.ts = size # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE, see _split_delta + + if d.has_copy_chunklist(): + d.data.set_rbound(size) + # END truncate chunklist + + return d def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static - :param bytes: amount of bytes to move, must be smaller than delta size""" + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return if bytes >= d.ts: raise ValueError("Cannot move offset that much") d.to += bytes d.so += bytes d.ts -= bytes - if d.data: - d.data = d.data[bytes:] + if d.data is not None: + if isinstance(d.data, DeltaChunkList): + d.data.move_lbound(bytes) + else: + d.data = d.data[bytes:] + # END handle data type # END handle data + return d class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -106,16 +123,22 @@ def rbound(self): def has_data(self): """:return: True if the instance has data to add to the target stream""" - return self.data is None or not isinstance(self.data, DeltaChunkList) + return self.data is not None and not isinstance(self.data, DeltaChunkList) - def apply(self, source, write): + def has_copy_chunklist(self): + """:return: True if we copy our data from a chunklist""" + return return self.data is not None and isinstance(self.data, DeltaChunkList) + + def apply(self, bbuf, write): """Apply own data to the target buffer - :param source: buffer providing source bytes for copy operations + :param bbuf: buffer providing source bytes for copy operations :param write: write method to call with data to write""" - if self.has_data(): + if self.data is None: # COPY DATA FROM SOURCE - assert len(source) - self.so - self.ts > 0 - write(buffer(source, self.so, self.ts)) + assert len(bbuf) - self.so - self.ts > 0 + write(buffer(bbuf, self.so, self.ts)) + elif isinstance(self.data, DeltaChunkList): + self.data.apply(bbuf, write) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -153,6 +176,8 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): :note: belongs to DeltaChunkList""" if relofs > d.ts: raise ValueError("Cannot split behinds a chunks rbound") + if relofs < 1: + raise ValueError("Cannot split delta with %i" % relofs) osize = d.ts - relofs _set_delta_rbound(d, relofs) @@ -295,23 +320,65 @@ def init(self, size): return self - def set_rbound(self, size): - """Chops the list at the given size, splitting and removing DeltaNodes + def set_rbound(self, relofs): + """Chops the list at the given relative offset, splitting and removing DeltaNodes as required + :param relofs: offset relative to the start of the chain :return: self""" - di = _closest_index(self, size) + if len(self) == 0: + raise AssertionError("Cannot change bound of empty list") + if relofs == 0: + raise ValueError("Size to truncate to must not be 0") + absofs = self.lbound() + relofs + if absofs > self.rbound(): + raise ValueError("Cannot extend chunk list") + di = _closest_index(self, absofs) d = self[di] - rsize = size - d.to + rsize = absofs - d.to if rsize: _set_delta_rbound(d, rsize) # END truncate last node if possible del(self[di+(rsize!=0):]) ## DEBUG ## - self.check_integrity(size) + self.check_integrity(absofs) return self + def move_lbound(self, bytes): + """Offset the left bound of the list by the given amount of bytes. + This effectively truncates the list + :return: self""" + if len(self) == 0: + raise AssertionError("Cannot change bound of empty list") + if bytes == 0: + return + abslbound = self.lbound() + bytes + if abslbound >= self.rbound(): + raise ValueError("Cannot move lbound that much") + + dsi = _closest_index(self, abslbound) + d = self[dsi] + _move_delta_lbound(d, abslbound - d.to) + + if dsi: + del(self[:dsi]) + # END remove all skipped nodes + + return self + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + def connect_with(self, bdlc): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region @@ -319,7 +386,11 @@ def connect_with(self, bdlc): will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. :param bdlc: DeltaChunkList to serve as base""" - raise NotImplementedError("todo") + for dc in self: + if not dc.has_data(): + dc.data = bdcl[dc.to, dc.ts] + # END handle overlap + # END for each dc def apply(self, bbuf, write): """Apply the chain's changes and write the final result using the passed @@ -328,7 +399,10 @@ def apply(self, bbuf, write): list. It will only be used if the chunk in question does not have a base chain. :param write: function taking a string of bytes to write to the output""" - raise NotImplementedError("todo") + dapply = DeltaChunk.apply + for dc in self: + dapply(dc, bbuf, write) + # END for each dc def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches @@ -345,6 +419,7 @@ def check_integrity(self, target_size=-1): # check data for dc in self: + assert dc.ts > 0 if dc.data: assert len(dc.data) >= dc.ts # END for each dc @@ -359,6 +434,77 @@ def check_integrity(self, target_size=-1): assert lft.to + lft.ts == rgt.to # END for each pair + def __getslice__(self, absofs, size): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: DeltaChunkList (copy) which represents the given chunk""" + cdi = _closest_index(self, absofs) # delta start index + slen = len(self) + ndcl = self.__class__() + rbound = absofs + size + + while cdi < slen: + # are we larger than the current block + cd = self[cdi] + if absofs < cd.to: + if rbound >= cd.rbound(): + # xxx|xxx|x + # cd is fully contained in the range + ndcl.append(copy(cd)) + elif rbound > cd.to: + # partially contained + # xxx|x--| + cd = copy(cd) + _set_delta_rbound(cd, cd.rbound() - rbound) + ndcl.append(cd) + break + else: + # xx.|---| + # WE DON'T OVERLAP IT + break + # END rbound overlap handling + # END lbound overlap handling + else: + if absofs >= cd.rbound(): + # happens if slice is out of bound + #|---|xx + break + # END + + if rbound >= cd.rbound(): + if absofs == cd.to: + #|xxx|x + # fully contained + ndcl.append(copy(cd)) + else: + # shift + #|-xx| + cd = copy(cd) + _move_delta_lbound(cd, absofs - cd.to) + ndcl.append(cd) + # END handle offset special case + elif absofs == cd.to: + #|x--| + # we truncate it to our size + cd = copy(cd) + _set_delta_rbound(cd, size) + ndcl.append(cd) + break + else: + #|-x-| + # adjust both ends + cd = copy(cd) + _move_delta_lbound(cd, absofs - cd.to) + _set_delta_rbound(cd, size) + ndcl.append(cd) + break + # END handle rbound overlap + # END handle overlap + # END for each chunk + return ndcl + + + #} END structures #{ Routines @@ -559,16 +705,16 @@ def merge_deltas(dstreams): # END while processing delta data # merge the lists ! - if base is not None: - dcl.connect_with(base) + if bdcl is not None: + dcl.connect_with(bdcl) # END handle merge # prepare next base - base = dcl + bdcl = dcl dcl = DeltaChunkList() # END for each delta stream - # print dcl + return base def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): From 1c2caf590d85866d95c3d1470eba692a61de3622 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 19:37:22 +0200 Subject: [PATCH 0064/3719] Forward Delta Application now appears to work --- fun.py | 419 +++++++++++++++++------------------------------------- stream.py | 18 ++- 2 files changed, 146 insertions(+), 291 deletions(-) diff --git a/fun.py b/fun.py index 3e126c3e4..ac0b1c098 100644 --- a/fun.py +++ b/fun.py @@ -44,8 +44,8 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'reverse_merge_deltas', - 'merge_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'reverse_connect_deltas', + 'connect_deltas', 'DeltaChunkList') #{ Structures @@ -55,21 +55,17 @@ def _set_delta_rbound(d, size): :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size :return: d""" - if size == 0: - raise ValueError("size to truncate to must not be 0") if d.ts == size: return + if size == 0: + raise ValueError("size to truncate to must not be 0") if size > d.ts: raise ValueError("Cannot extend rbound") d.ts = size # NOTE: data is truncated automatically when applying the delta - # MUST NOT DO THIS HERE, see _split_delta - - if d.has_copy_chunklist(): - d.data.set_rbound(size) - # END truncate chunklist + # MUST NOT DO THIS HERE return d @@ -85,13 +81,10 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes + d.sob += bytes d.ts -= bytes - if d.data is not None: - if isinstance(d.data, DeltaChunkList): - d.data.move_lbound(bytes) - else: - d.data = d.data[bytes:] - # END handle data type + if d.has_data(): + d.data = d.data[bytes:] # END handle data return d @@ -103,14 +96,16 @@ class DeltaChunk(object): 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None - 'data' # chunk of bytes to be added to the target buffer, + 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None + 'sob' # DEBUG: Backup ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts - self.so = sos + self.so = so + self.sob = so self.data = data def __repr__(self): @@ -118,6 +113,18 @@ def __repr__(self): #{ Interface + def copy_offset(self): + """:return: offset to apply when copying from a base buffer, or 0 + if this is not a copying delta chunk""" + + if self.data is not None: + if isinstance(self.data, DeltaChunkList): + return self.data.lbound() + self.so + else: + return self.so + # END handle data type + return 0 + def rbound(self): return self.to + self.ts @@ -127,7 +134,15 @@ def has_data(self): def has_copy_chunklist(self): """:return: True if we copy our data from a chunklist""" - return return self.data is not None and isinstance(self.data, DeltaChunkList) + return self.data is not None and isinstance(self.data, DeltaChunkList) + + def set_copy_chunklist(self, dcl): + """Set the deltachunk list to be used as basis for copying. + :note: only works if this chunk is a copy delta chunk""" + assert self.data is None, "Cannot assign chain to add delta chunk" + self.data = dcl + self.sob = self.so + self.so = 0 # allows lbound moves to be virtual def apply(self, bbuf, write): """Apply own data to the target buffer @@ -135,10 +150,10 @@ def apply(self, bbuf, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE - assert len(bbuf) - self.so - self.ts > 0 + assert len(bbuf) - self.so - self.ts > -1 write(buffer(bbuf, self.so, self.ts)) elif isinstance(self.data, DeltaChunkList): - self.data.apply(bbuf, write) + self.data.apply(bbuf, write, self.so, self.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -165,208 +180,10 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 -def _split_delta(dcl, d, di, relofs, insert_offset=0): - """Split the delta at di into two deltas, adjusting their sizes, offsets and data - accordingly and adding the new part to the dcl - :param relofs: relative offset at which to split the delta - :param d: delta chunk to split - :param di: index of d in dcl - :param insert_offset: offset for the new split id - :return: newly created DeltaChunk - :note: belongs to DeltaChunkList""" - if relofs > d.ts: - raise ValueError("Cannot split behinds a chunks rbound") - if relofs < 1: - raise ValueError("Cannot split delta with %i" % relofs) - - osize = d.ts - relofs - _set_delta_rbound(d, relofs) - - # insert new one - drb = d.rbound() - - nd = DeltaChunk( drb, - osize, - d.so + osize, - (d.data and d.data[osize:]) or None ) - - self.insert(di+1+insert_offset, nd) - return nd - -def _handle_merge(ld, rd): - """Optimize the layout of the lhs delta and the rhs delta - TODO: Once the default implementation is working""" - if d.has_data(): - if od.data: - # OVERWRITE DATA - pass - else: - # MERGE SOURCE AREA - pass - # END overwrite data - else: - if od.data: - # MERGE DATA WITH DATA - # overwrite the data at the respective spot - pass - else: - # INSERT DATA INTO COPY AREA - pass - # END combine or insert data - # END handle chunk mode - -def _merge_delta(dcl, dc): - """Merge the given DeltaChunk instance into the dcl - :param d: the DeltaChunk to merge""" - if len(dcl) == 0: - dcl.append(dc) - return - # END early return on empty list - - cdi = _closest_index(dcl, dc.to) # current delta index - cd = dcl[cdi] # current delta - - # either we go at his spot, or after - # cdi either moves one up, or stays - #print "insert at %i" % (cdi + (dc.to > cd.to)) - #print cd, dc - dcl.insert(cdi + (dc.to > cd.to), dc) - cdi += dc.to == cd.to - - while True: - # are we larger than the current block - if dc.to < cd.to: - if dc.rbound() >= cd.rbound(): - # xxx|xxx|x - # remove the current item completely - dcl.pop(cdi) - cdi -= 1 - elif dc.rbound() > cd.to: - # MOVE ITS LBOUND - # xxx|x--| - _move_delta_lbound(cd, dc.rbound() - cd.to) - break - else: - # xx.|---| - # WE DON'T OVERLAP IT - # this can actually happen, once multiple streams are merged - break - # END rbound overlap handling - # END lbound overlap handling - else: - if dc.to >= cd.rbound(): - #|---|xx - break - # END - - if dc.rbound() >= cd.rbound(): - if dc.to == cd.to: - #|xxx|x - # REMOVE CD - dcl.pop(cdi) - cdi -= 1 - else: - # TRUNCATE CD - #|-xx| - _set_delta_rbound(cd, dc.to - cd.to) - # END handle offset special case - elif dc.to == cd.to: - #|x--| - # we shift it by our size - _move_delta_lbound(cd, dc.ts) - else: - #|-x-| - # SPLIT CD AND LBOUND MOVE ITS SECOND PART - # insert offset is required to insert it after us - nd = _split_delta(dcl, cd, cdi, 1) - _move_delta_lbound(nd, dc.ts) - break - # END handle rbound overlap - # END handle overlap - - cdi += 1 - if cdi < len(dcl): - cd = dcl[cdi] - else: - break - # END check for end of list - # while our chunk is not completely done - - ## DEBUG ## - dcl.check_integrity() - - class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def init(self, size): - """Intialize this instance with chunks defining to fill up size from a base - buffer of equal size - :return: self""" - if len(self) != 0: - return - # pretend we have one huge delta chunk, which just copies everything - # from source to destination - maxint32 = 2**32 - for x in range(0, size, maxint32): - self.append(DeltaChunk(x, maxint32, x, None)) - # END create copy chunks - offset = x*maxint32 - remainder = size-offset - if remainder: - self.append(DeltaChunk(offset, remainder, offset, None)) - # END handle all done in loop - - return self - - def set_rbound(self, relofs): - """Chops the list at the given relative offset, splitting and removing DeltaNodes - as required - :param relofs: offset relative to the start of the chain - :return: self""" - if len(self) == 0: - raise AssertionError("Cannot change bound of empty list") - if relofs == 0: - raise ValueError("Size to truncate to must not be 0") - absofs = self.lbound() + relofs - if absofs > self.rbound(): - raise ValueError("Cannot extend chunk list") - di = _closest_index(self, absofs) - d = self[di] - rsize = absofs - d.to - if rsize: - _set_delta_rbound(d, rsize) - # END truncate last node if possible - del(self[di+(rsize!=0):]) - - ## DEBUG ## - self.check_integrity(absofs) - - return self - - def move_lbound(self, bytes): - """Offset the left bound of the list by the given amount of bytes. - This effectively truncates the list - :return: self""" - if len(self) == 0: - raise AssertionError("Cannot change bound of empty list") - if bytes == 0: - return - abslbound = self.lbound() + bytes - if abslbound >= self.rbound(): - raise ValueError("Cannot move lbound that much") - - dsi = _closest_index(self, abslbound) - d = self[dsi] - _move_delta_lbound(d, abslbound - d.to) - - if dsi: - del(self[:dsi]) - # END remove all skipped nodes - - return self - def rbound(self): """:return: rightmost extend in bytes, absolute""" if len(self) == 0: @@ -379,30 +196,84 @@ def lbound(self): return 0 return self[0].to - def connect_with(self, bdlc): + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def connect_with(self, bdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdlc: DeltaChunkList to serve as base""" + :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - dc.data = bdcl[dc.to, dc.ts] + # dc.set_copy_chunklist(bdcl[dc.copy_offset():dc.ts]) + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) # END handle overlap # END for each dc - def apply(self, bbuf, write): + def apply(self, bbuf, write, lbound_offset=0, size=0): """Apply the chain's changes and write the final result using the passed write function. :param bbuf: base buffer containing the base of all deltas contained in this list. It will only be used if the chunk in question does not have a base chain. + :param lbound_offset: offset at which to start applying the delta, relative to + our lbound + :param size: if larger than 0, only the given amount of bytes will be applied :param write: function taking a string of bytes to write to the output""" + slen = len(self) + if slen == 0: + return + # END early abort + absofs = self.lbound() + lbound_offset + if size == 0: + size = self.rbound() - absofs + # END initialize size + if absofs + size > self.rbound(): + raise ValueError("Cannot apply more bytes than there are in this chain") + # END sanity check + + if size > self.rbound() - absofs: + raise ValueError("Trying to apply more than there is available") + dapply = DeltaChunk.apply - for dc in self: - dapply(dc, bbuf, write) - # END for each dc + if lbound_offset or absofs + size != self.rbound(): + cdi = _closest_index(self, absofs) + cd = self[cdi] + if cd.to != absofs: + tcd = copy(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + dapply(tcd, bbuf, write) + size -= tcd.ts + cdi += 1 + # END handle first chunk + + # here we have to either apply full chunks, or smaller ones, but + # we always start at the chunks target offset + while cdi < slen and size: + cd = self[cdi] + if cd.ts <= size: + dapply(cd, bbuf, write) + size -= cd.ts + else: + tcd = copy(cd) + _set_delta_rbound(tcd, size) + dapply(tcd, bbuf, write) + size -= tcd.ts + break + # END handle bytes to apply + cdi += 1 + # END handle rest + assert size == 0 + else: + for dc in self: + dapply(dc, bbuf, write) + # END for each dc + # END handle application values def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches @@ -420,8 +291,10 @@ def check_integrity(self, target_size=-1): # check data for dc in self: assert dc.ts > 0 - if dc.data: + if dc.has_data(): assert len(dc.data) >= dc.ts + if dc.has_copy_chunklist(): + assert dc.ts <= dc.data.size() # END for each dc left = islice(self, 0, len(self)-1) @@ -438,69 +311,43 @@ def __getslice__(self, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: DeltaChunkList (copy) which represents the given chunk""" + if len(self) == 0: + return DeltaChunkList() + + absofs = max(absofs, self.lbound()) + size = min(self.rbound() - self.lbound(), size) cdi = _closest_index(self, absofs) # delta start index + cd = self[cdi] slen = len(self) ndcl = self.__class__() - rbound = absofs + size - while cdi < slen: + if cd.to != absofs: + tcd = copy(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + ndcl.append(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: # are we larger than the current block cd = self[cdi] - if absofs < cd.to: - if rbound >= cd.rbound(): - # xxx|xxx|x - # cd is fully contained in the range - ndcl.append(copy(cd)) - elif rbound > cd.to: - # partially contained - # xxx|x--| - cd = copy(cd) - _set_delta_rbound(cd, cd.rbound() - rbound) - ndcl.append(cd) - break - else: - # xx.|---| - # WE DON'T OVERLAP IT - break - # END rbound overlap handling - # END lbound overlap handling + if cd.ts <= size: + ndcl.append(copy(cd)) + size -= cd.ts else: - if absofs >= cd.rbound(): - # happens if slice is out of bound - #|---|xx - break - # END - - if rbound >= cd.rbound(): - if absofs == cd.to: - #|xxx|x - # fully contained - ndcl.append(copy(cd)) - else: - # shift - #|-xx| - cd = copy(cd) - _move_delta_lbound(cd, absofs - cd.to) - ndcl.append(cd) - # END handle offset special case - elif absofs == cd.to: - #|x--| - # we truncate it to our size - cd = copy(cd) - _set_delta_rbound(cd, size) - ndcl.append(cd) - break - else: - #|-x-| - # adjust both ends - cd = copy(cd) - _move_delta_lbound(cd, absofs - cd.to) - _set_delta_rbound(cd, size) - ndcl.append(cd) - break - # END handle rbound overlap - # END handle overlap + tcd = copy(cd) + _set_delta_rbound(tcd, size) + ndcl.append(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 # END for each chunk + assert size == 0, "size was %i" % size + + ndcl.check_integrity() return ndcl @@ -627,7 +474,7 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw -def reverse_merge_deltas(dcl, dstreams): +def reverse_connect_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dcl: see 3 @@ -636,7 +483,7 @@ def reverse_merge_deltas(dcl, dstreams): :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def merge_deltas(dstreams): +def connect_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dstreams: iterable of delta stream objects. They must be ordered latest last, @@ -690,12 +537,10 @@ def merge_deltas(dstreams): rbound > base_size): break - # _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - # _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c @@ -709,12 +554,14 @@ def merge_deltas(dstreams): dcl.connect_with(bdcl) # END handle merge + dcl.check_integrity() + # prepare next base bdcl = dcl dcl = DeltaChunkList() # END for each delta stream - return base + return bdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): diff --git a/stream.py b/stream.py index efb99d218..8b8655e86 100644 --- a/stream.py +++ b/stream.py @@ -8,7 +8,7 @@ msb_size, stream_copy, apply_delta_data, - merge_deltas, + connect_deltas, DeltaChunkList, delta_types ) @@ -325,7 +325,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = merge_deltas(reversed(self._dstreams)) + dcl = connect_deltas(reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -333,7 +333,7 @@ def _set_cache_(self, attr): return # END handle empty list - self._size = dcl[-1].rbound() + self._size = dcl.rbound() self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) @@ -353,8 +353,16 @@ def _set_cache_(self, attr): self._set_cache_old(attr) import chardet - if chardet.detect(mt[:])['encoding'] == 'ascii': - assert self._mm_target[:] == mt[:] + + print "num dstreams", len(self._dstreams) + #if chardet.detect(mt[:self._size])['encoding'] == 'ascii': + if self._mm_target[:self._size] != mt[:]: + open("working.txt", "w").write(self._mm_target[:self._size]) + open("incorrect.txt", "w").write(mt[:]) + raise AssertionError("Output didn't match") + # END debug + print "success" + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" From 834f081232cb51c251e8f2d3931c9e1fd5eff457 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:07:28 +0200 Subject: [PATCH 0065/3719] Implemented add-chunk compression, which clearly reduces chain size, but might not really be worth it in python --- fun.py | 70 +++++++++++++++++++++++++++++++++++++++++-------------- stream.py | 29 +++++++---------------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/fun.py b/fun.py index ac0b1c098..170b3db15 100644 --- a/fun.py +++ b/fun.py @@ -13,6 +13,7 @@ from itertools import islice, izip from copy import copy +from cStringIO import StringIO # INVARIANTS OFS_DELTA = 6 @@ -57,10 +58,6 @@ def _set_delta_rbound(d, size): :return: d""" if d.ts == size: return - if size == 0: - raise ValueError("size to truncate to must not be 0") - if size > d.ts: - raise ValueError("Cannot extend rbound") d.ts = size @@ -76,8 +73,6 @@ def _move_delta_lbound(d, bytes): :return: d""" if bytes == 0: return - if bytes >= d.ts: - raise ValueError("Cannot move offset that much") d.to += bytes d.so += bytes @@ -139,7 +134,6 @@ def has_copy_chunklist(self): def set_copy_chunklist(self, dcl): """Set the deltachunk list to be used as basis for copying. :note: only works if this chunk is a copy delta chunk""" - assert self.data is None, "Cannot assign chain to add delta chunk" self.data = dcl self.sob = self.so self.so = 0 # allows lbound moves to be virtual @@ -150,13 +144,13 @@ def apply(self, bbuf, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE - assert len(bbuf) - self.so - self.ts > -1 write(buffer(bbuf, self.so, self.ts)) elif isinstance(self.data, DeltaChunkList): self.data.apply(bbuf, write, self.so, self.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it if self.ts < len(self.data): write(self.data[:self.ts]) else: @@ -209,11 +203,54 @@ def connect_with(self, bdcl): :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - # dc.set_copy_chunklist(bdcl[dc.copy_offset():dc.ts]) dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) # END handle overlap # END for each dc + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + slen_orig = slen + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if not dc.has_data(): + if first_data_index is not None and i-2-first_data_index > 1: + #if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in xrange(first_data_index, i-1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i-1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i-1 + # END iterate list + + #if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + def apply(self, bbuf, write, lbound_offset=0, size=0): """Apply the chain's changes and write the final result using the passed write function. @@ -232,12 +269,6 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): if size == 0: size = self.rbound() - absofs # END initialize size - if absofs + size > self.rbound(): - raise ValueError("Cannot apply more bytes than there are in this chain") - # END sanity check - - if size > self.rbound() - absofs: - raise ValueError("Trying to apply more than there is available") dapply = DeltaChunk.apply if lbound_offset or absofs + size != self.rbound(): @@ -347,7 +378,7 @@ def __getslice__(self, absofs, size): # END for each chunk assert size == 0, "size was %i" % size - ndcl.check_integrity() + # ndcl.check_integrity() return ndcl @@ -540,7 +571,8 @@ def connect_deltas(dstreams): dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: - # TODO: Concatenate multiple deltachunks + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c @@ -549,12 +581,14 @@ def connect_deltas(dstreams): # END handle command byte # END while processing delta data + dcl.compress() + # merge the lists ! if bdcl is not None: dcl.connect_with(bdcl) # END handle merge - dcl.check_integrity() + # dcl.check_integrity() # prepare next base bdcl = dcl diff --git a/stream.py b/stream.py index 8b8655e86..098d27a2b 100644 --- a/stream.py +++ b/stream.py @@ -322,6 +322,14 @@ def __init__(self, stream_list): self._br = 0 def _set_cache_(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python + #print "num streams", len(self._dstreams) + #if len(self._dstreams) == 1 or (len(self._dstreams) * self._dstreams.size) > 25*1000*1000: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. @@ -345,26 +353,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - ## DEBUG ## - mt = self._mm_target - for ds in self._dstreams: - ds.stream.seek(0) - self._bstream.stream.seek(0) - self._set_cache_old(attr) - - import chardet - - print "num dstreams", len(self._dstreams) - #if chardet.detect(mt[:self._size])['encoding'] == 'ascii': - if self._mm_target[:self._size] != mt[:]: - open("working.txt", "w").write(self._mm_target[:self._size]) - open("incorrect.txt", "w").write(mt[:]) - raise AssertionError("Output didn't match") - # END debug - print "success" - - - def _set_cache_old(self, attr): + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From bda5ef5e94161c304d6151785b34a20bdb306389 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:32:01 +0200 Subject: [PATCH 0066/3719] implemented binary tree search to get the closest deltachunk by offset --- fun.py | 18 ++++++++++++------ test/test_pack.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index 170b3db15..8f38fa467 100644 --- a/fun.py +++ b/fun.py @@ -165,12 +165,18 @@ def _closest_index(dcl, absofs): to the DeltaChunk with a target buffer absofs that equals or is greater than absofs. :note: global method for performance only, it belongs to DeltaChunkList""" - # TODO: binary search !! - for i,d in enumerate(dcl): - if absofs < d.to: - return i-1 - elif absofs == d.to: - return i + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound # END for each delta absofs return len(dcl)-1 diff --git a/test/test_pack.py b/test/test_pack.py index 770a78bad..6e598d755 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def test_pack_entity(self): + def _test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): From 5d18685948602de69eb950d0238fcf80f0413b68 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:53:11 +0200 Subject: [PATCH 0067/3719] Disabled delta-aggregation as it is reduces the throughput to 540KiB/s compared to 9.4MiB compared to the previous brute-force algorithm. Compression helps, but it would probably be more efficient if done right away, not as post-process. It might help to implement the reversed version of this algorithm, as initially intended, but currently the overhead is the actual application --- fun.py | 4 ---- stream.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index 8f38fa467..bc67e0fce 100644 --- a/fun.py +++ b/fun.py @@ -76,7 +76,6 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes - d.sob += bytes d.ts -= bytes if d.has_data(): d.data = d.data[bytes:] @@ -93,14 +92,12 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - 'sob' # DEBUG: Backup ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts self.so = so - self.sob = so self.data = data def __repr__(self): @@ -135,7 +132,6 @@ def set_copy_chunklist(self, dcl): """Set the deltachunk list to be used as basis for copying. :note: only works if this chunk is a copy delta chunk""" self.data = dcl - self.sob = self.so self.so = 0 # allows lbound moves to be virtual def apply(self, bbuf, write): diff --git a/stream.py b/stream.py index 098d27a2b..7347f527c 100644 --- a/stream.py +++ b/stream.py @@ -311,6 +311,10 @@ class DeltaApplyReader(LazyMixin): "_br" # number of bytes read ) + #{ Configuration + k_max_memory_move = 250*1000*1000 + #} END configuration + def __init__(self, stream_list): """Initialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the @@ -325,8 +329,9 @@ def _set_cache_(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python - #print "num streams", len(self._dstreams) - #if len(self._dstreams) == 1 or (len(self._dstreams) * self._dstreams.size) > 25*1000*1000: + # hence we apply a worst-case scenario here + # TODO: read the final size from the deltastream - have to partly unpack + # if len(self._dstreams) * self._size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) @@ -353,7 +358,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_brute_(self, attr): + def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From 682f483fa61c77fd6121ae860002094eb517bbd4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 00:07:37 +0200 Subject: [PATCH 0068/3719] First profiling run revealed that the copy function was a serious slowdown. Now its twice as fast compared to the previous version, but still about 8 times slower than the brute force approach --- fun.py | 14 ++++++++------ stream.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index bc67e0fce..9f2c9b6be 100644 --- a/fun.py +++ b/fun.py @@ -12,7 +12,6 @@ import mmap from itertools import islice, izip -from copy import copy from cStringIO import StringIO # INVARIANTS @@ -82,6 +81,9 @@ def _move_delta_lbound(d, bytes): # END handle data return d + +def delta_duplicate(src): + return DeltaChunk(src.to, src.ts, src.so, src.data) class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -277,7 +279,7 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): cdi = _closest_index(self, absofs) cd = self[cdi] if cd.to != absofs: - tcd = copy(cd) + tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) dapply(tcd, bbuf, write) @@ -293,7 +295,7 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): dapply(cd, bbuf, write) size -= cd.ts else: - tcd = copy(cd) + tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) dapply(tcd, bbuf, write) size -= tcd.ts @@ -355,7 +357,7 @@ def __getslice__(self, absofs, size): ndcl = self.__class__() if cd.to != absofs: - tcd = copy(cd) + tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) ndcl.append(tcd) @@ -367,10 +369,10 @@ def __getslice__(self, absofs, size): # are we larger than the current block cd = self[cdi] if cd.ts <= size: - ndcl.append(copy(cd)) + ndcl.append(delta_duplicate(cd)) size -= cd.ts else: - tcd = copy(cd) + tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) ndcl.append(tcd) size -= tcd.ts diff --git a/stream.py b/stream.py index 7347f527c..16c917a97 100644 --- a/stream.py +++ b/stream.py @@ -358,7 +358,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_(self, attr): + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From a064007f0e25b8a4af04c30fe329e345edc8092e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 08:08:09 +0200 Subject: [PATCH 0069/3719] Made heavliy called methods global, its brings a second, which is nearly 10 percent more performance just by eliminating two method calls --- fun.py | 154 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 79 insertions(+), 75 deletions(-) diff --git a/fun.py b/fun.py index 9f2c9b6be..253e64092 100644 --- a/fun.py +++ b/fun.py @@ -84,6 +84,26 @@ def _move_delta_lbound(d, bytes): def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) + +def delta_chunk_apply(dc, bbuf, write): + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(buffer(bbuf, dc.so, dc.ts)) + elif isinstance(dc.data, DeltaChunkList): + delta_list_apply(dc.data, bbuf, write, dc.so, dc.ts) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -136,25 +156,7 @@ def set_copy_chunklist(self, dcl): self.data = dcl self.so = 0 # allows lbound moves to be virtual - def apply(self, bbuf, write): - """Apply own data to the target buffer - :param bbuf: buffer providing source bytes for copy operations - :param write: write method to call with data to write""" - if self.data is None: - # COPY DATA FROM SOURCE - write(buffer(bbuf, self.so, self.ts)) - elif isinstance(self.data, DeltaChunkList): - self.data.apply(bbuf, write, self.so, self.ts) - else: - # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? - # Considering data can be larger than 127 bytes now, it should be worth it - if self.ts < len(self.data): - write(self.data[:self.ts]) - else: - write(self.data) - # END handle truncation - # END handle chunk mode + #} END interface @@ -178,6 +180,59 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 +def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param lbound_offset: offset at which to start applying the delta, relative to + our lbound + :param size: if larger than 0, only the given amount of bytes will be applied + :param write: function taking a string of bytes to write to the output""" + slen = len(dcl) + if slen == 0: + return + # END early abort + absofs = dcl.lbound() + lbound_offset + if size == 0: + size = dcl.rbound() - absofs + # END initialize size + + if lbound_offset or absofs + size != dcl.rbound(): + cdi = _closest_index(dcl, absofs) + cd = dcl[cdi] + if cd.to != absofs: + tcd = delta_duplicate(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + delta_chunk_apply(tcd, bbuf, write) + size -= tcd.ts + cdi += 1 + # END handle first chunk + + # here we have to either apply full chunks, or smaller ones, but + # we always start at the chunks target offset + while cdi < slen and size: + cd = dcl[cdi] + if cd.ts <= size: + delta_chunk_apply(cd, bbuf, write) + size -= cd.ts + else: + tcd = delta_duplicate(cd) + _set_delta_rbound(tcd, size) + delta_chunk_apply(tcd, bbuf, write) + size -= tcd.ts + break + # END handle bytes to apply + cdi += 1 + # END handle rest + else: + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc + # END handle application values + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" @@ -211,6 +266,11 @@ def connect_with(self, bdcl): # END handle overlap # END for each dc + def apply(self, bbuf, write, lbound_offset=0, size=0): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write, lbound_offset, size) + def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate add-chunks @@ -255,61 +315,6 @@ def compress(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self - def apply(self, bbuf, write, lbound_offset=0, size=0): - """Apply the chain's changes and write the final result using the passed - write function. - :param bbuf: base buffer containing the base of all deltas contained in this - list. It will only be used if the chunk in question does not have a base - chain. - :param lbound_offset: offset at which to start applying the delta, relative to - our lbound - :param size: if larger than 0, only the given amount of bytes will be applied - :param write: function taking a string of bytes to write to the output""" - slen = len(self) - if slen == 0: - return - # END early abort - absofs = self.lbound() + lbound_offset - if size == 0: - size = self.rbound() - absofs - # END initialize size - - dapply = DeltaChunk.apply - if lbound_offset or absofs + size != self.rbound(): - cdi = _closest_index(self, absofs) - cd = self[cdi] - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - dapply(tcd, bbuf, write) - size -= tcd.ts - cdi += 1 - # END handle first chunk - - # here we have to either apply full chunks, or smaller ones, but - # we always start at the chunks target offset - while cdi < slen and size: - cd = self[cdi] - if cd.ts <= size: - dapply(cd, bbuf, write) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - dapply(tcd, bbuf, write) - size -= tcd.ts - break - # END handle bytes to apply - cdi += 1 - # END handle rest - assert size == 0 - else: - for dc in self: - dapply(dc, bbuf, write) - # END for each dc - # END handle application values - def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -380,7 +385,6 @@ def __getslice__(self, absofs, size): # END hadle size cdi += 1 # END for each chunk - assert size == 0, "size was %i" % size # ndcl.check_integrity() return ndcl From e814fda2ef0b2de172fe06c2cbd1bc539a368262 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 12:04:59 +0200 Subject: [PATCH 0070/3719] First frame to implement the actual data aggregation, but ... its probbaly going to change quite a lot again --- fun.py | 46 +++++++++++++++++++++++++++++++--------------- stream.py | 2 +- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/fun.py b/fun.py index 253e64092..ec3662fe4 100644 --- a/fun.py +++ b/fun.py @@ -83,7 +83,7 @@ def _move_delta_lbound(d, bytes): return d def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data) + return DeltaChunk(src.to, src.ts, src.so, src.data, src.flags) def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer @@ -114,16 +114,18 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None + 'flags' # currently only True or False ) - def __init__(self, to, ts, so, data): + def __init__(self, to, ts, so, data, flags): self.to = to self.ts = ts self.so = so self.data = data + self.flags = flags def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + return "DeltaChunk(%i, %i, %s, %s, %i)" % (self.to, self.ts, self.so, self.data or "", self.flags) #{ Interface @@ -253,18 +255,24 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl): + def connect_with(self, bdcl, tdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base""" - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) - # END handle overlap - # END for each dc + :param bdcl: DeltaChunkList to serve as base + :param tdcl: topmost delta chunk list. If set, reverse order is assumed + and the list is connected more efficiently""" + if tdcl is None: + for dc in self: + if not dc.has_data(): + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + # END handle overlap + # END for each dc + else: + raise NotImplementedError("todo") + # END handle order def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines @@ -297,7 +305,7 @@ def compress(self): del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf, False)) slen = len(self) i = first_data_index + 1 @@ -522,14 +530,18 @@ def reverse_connect_deltas(dcl, dstreams): :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def connect_deltas(dstreams): +def connect_deltas(dstreams, reverse): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first + :param reverse: If False, the given iterable of delta-streams returns + items in from latest ancestor to the last delta. + If True, deltas are ordered so that the one to be applied last comes first. :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base dcl = DeltaChunkList() + tdcl = None # topmost dcl, only effective if reverse is True for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -576,12 +588,12 @@ def connect_deltas(dstreams): rbound > base_size): break - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None, False)) tbw += cp_size elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c], False)) i += c tbw += c else: @@ -591,9 +603,13 @@ def connect_deltas(dstreams): dcl.compress() + if reverse and tdcl is None: + tdcl = dcl + # END handle reverse + # merge the lists ! if bdcl is not None: - dcl.connect_with(bdcl) + dcl.connect_with(bdcl, tdcl) # END handle merge # dcl.check_integrity() diff --git a/stream.py b/stream.py index 16c917a97..40a0c6c6a 100644 --- a/stream.py +++ b/stream.py @@ -338,7 +338,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = connect_deltas(reversed(self._dstreams)) + dcl = connect_deltas(self._dstreams, reverse=True) if len(dcl) == 0: self._size = 0 From 2e19424354d03a5351ac374dc8dd36f6b65b0d5e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 12:33:14 +0200 Subject: [PATCH 0071/3719] New Frame uses a distinct type to express the different mode of operation. This is clean enough to get going --- fun.py | 85 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/fun.py b/fun.py index ec3662fe4..05925721c 100644 --- a/fun.py +++ b/fun.py @@ -237,7 +237,13 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): class DeltaChunkList(list): - """List with special functionality to deal with DeltaChunks""" + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() def rbound(self): """:return: rightmost extend in bytes, absolute""" @@ -255,24 +261,18 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl, tdcl): + def connect_with(self, bdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base - :param tdcl: topmost delta chunk list. If set, reverse order is assumed - and the list is connected more efficiently""" - if tdcl is None: - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) - # END handle overlap - # END for each dc - else: - raise NotImplementedError("todo") - # END handle order + :param bdcl: DeltaChunkList to serve as base""" + for dc in self: + if not dc.has_data(): + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + # END handle overlap + # END for each dc def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines @@ -396,8 +396,37 @@ def __getslice__(self, absofs, size): # ndcl.check_integrity() return ndcl - - + + +class TopdownDeltaChunkList(DeltaChunkList): + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = ('frozen', ) # if True, the list is frozen and can reproduce all data + # Will only be set in lists which where processed top-down + + def __init__(self): + self.frozen = False + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consequtively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams""" + if self.frozen == 1: + # Can that ever be hit ? + return False + # END early abort + # mark us so that the is_reversed method returns True, without us thinking + # we are frozen + self.frozen = -1 + + raise NotImplementedError("todo") + return True + #} END structures @@ -540,8 +569,14 @@ def connect_deltas(dstreams, reverse): If True, deltas are ordered so that the one to be applied last comes first. :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base - dcl = DeltaChunkList() tdcl = None # topmost dcl, only effective if reverse is True + + if reverse: + dcl = tdcl = TopdownDeltaChunkList() + else: + dcl = DeltaChunkList() + # END handle type of first chunk list + for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -603,13 +638,14 @@ def connect_deltas(dstreams, reverse): dcl.compress() - if reverse and tdcl is None: - tdcl = dcl - # END handle reverse - # merge the lists ! if bdcl is not None: - dcl.connect_with(bdcl, tdcl) + if tdcl: + if not tdcl.connect_with_next_base(dcl): + break + # END early abort + else: + dcl.connect_with(bdcl) # END handle merge # dcl.check_integrity() @@ -619,7 +655,10 @@ def connect_deltas(dstreams, reverse): dcl = DeltaChunkList() # END for each delta stream - return bdcl + if tdcl: + return tdcl + else: + return bdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): From f6bd67ce92257c9b5191b58de960cedda5159778 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 14:56:07 +0200 Subject: [PATCH 0072/3719] Reverse delta aggregration appears to be working --- fun.py | 155 +++++++++++++++++++++++++++++++--------------- test/test_pack.py | 2 +- 2 files changed, 107 insertions(+), 50 deletions(-) diff --git a/fun.py b/fun.py index 05925721c..9e4671a09 100644 --- a/fun.py +++ b/fun.py @@ -235,6 +235,46 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): # END for each dc # END handle application values +def delta_list_slice(dcl, absofs, size): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: DeltaChunkList (copy) which represents the given chunk""" + if len(dcl) == 0: + return DeltaChunkList() + + absofs = max(absofs, dcl.lbound()) + size = min(dcl.rbound() - dcl.lbound(), size) + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + ndcl = dcl.__class__() + + if cd.to != absofs: + tcd = delta_duplicate(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + ndcl.append(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + ndcl.append(delta_duplicate(cd)) + size -= cd.ts + else: + tcd = delta_duplicate(cd) + _set_delta_rbound(tcd, size) + ndcl.append(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + return ndcl class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. @@ -270,7 +310,7 @@ def connect_with(self, bdcl): :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts)) # END handle overlap # END for each dc @@ -355,49 +395,7 @@ def check_integrity(self, target_size=-1): assert lft.to + lft.ts == rgt.to # END for each pair - def __getslice__(self, absofs, size): - """:return: Subsection of this list at the given absolute offset, with the given - size in bytes. - :return: DeltaChunkList (copy) which represents the given chunk""" - if len(self) == 0: - return DeltaChunkList() - - absofs = max(absofs, self.lbound()) - size = min(self.rbound() - self.lbound(), size) - cdi = _closest_index(self, absofs) # delta start index - cd = self[cdi] - slen = len(self) - ndcl = self.__class__() - - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - ndcl.append(tcd) - size -= tcd.ts - cdi += 1 - # END lbound overlap handling - - while cdi < slen and size: - # are we larger than the current block - cd = self[cdi] - if cd.ts <= size: - ndcl.append(delta_duplicate(cd)) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - ndcl.append(tcd) - size -= tcd.ts - break - # END hadle size - cdi += 1 - # END for each chunk - - # ndcl.check_integrity() - return ndcl - class TopdownDeltaChunkList(DeltaChunkList): """Represents a list which is generated by feeding its ancestor streams one by one""" @@ -416,15 +414,76 @@ def connect_with_next_base(self, bdcl): consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams""" + assert self is not bdcl if self.frozen == 1: # Can that ever be hit ? return False # END early abort - # mark us so that the is_reversed method returns True, without us thinking - # we are frozen - self.frozen = -1 - raise NotImplementedError("todo") + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + sold = slen + while dci < slen: + dc = self[dci] + dci += 1 + + if dc.flags: + nfc += 1 + continue + # END skip frozen chunks + + # all data chunks must be frozen, we are topmost already + # (Also if its a copy operation onto the lowest base, but we cannot + # determine that without the number of deltas to come) + if dc.has_data(): + dc.flags = True + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + ccl = delta_list_slice(bdcl, dc.so, dc.ts) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + + assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound() + + if len(ccl) == 1: + self[dci-1] = ccl[0] + else: + + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci-1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl)-1 # deleted dc, added rest + + # END handle chunk replacement + + # END for each chunk + + if nfc == slen: + self.frozen = True + return False + # END handle completeness + return True @@ -648,8 +707,6 @@ def connect_deltas(dstreams, reverse): dcl.connect_with(bdcl) # END handle merge - # dcl.check_integrity() - # prepare next base bdcl = dcl dcl = DeltaChunkList() diff --git a/test/test_pack.py b/test/test_pack.py index 6e598d755..770a78bad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def _test_pack_entity(self): + def test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): From 0381cae63284e237a7023e9d40c7772690a2f84e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:11:02 +0200 Subject: [PATCH 0073/3719] Removed debugging code --- fun.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/fun.py b/fun.py index 9e4671a09..814c22a6a 100644 --- a/fun.py +++ b/fun.py @@ -399,11 +399,7 @@ def check_integrity(self, target_size=-1): class TopdownDeltaChunkList(DeltaChunkList): """Represents a list which is generated by feeding its ancestor streams one by one""" - __slots__ = ('frozen', ) # if True, the list is frozen and can reproduce all data - # Will only be set in lists which where processed top-down - - def __init__(self): - self.frozen = False + __slots__ = tuple() def connect_with_next_base(self, bdcl): """Connect this chain with the next level of our base delta chunklist. @@ -413,17 +409,10 @@ def connect_with_next_base(self, bdcl): :param bdcl: data chunk list being one of our bases. They must be fed in consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of - remaining streams""" - assert self is not bdcl - if self.frozen == 1: - # Can that ever be hit ? - return False - # END early abort - + remaining streams if False is returned""" nfc = 0 # number of frozen chunks dci = 0 # delta chunk index slen = len(self) # len of self - sold = slen while dci < slen: dc = self[dci] dci += 1 @@ -456,9 +445,6 @@ def connect_with_next_base(self, bdcl): cdc.to += ofs # END update target bounds - - assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound() - if len(ccl) == 1: self[dci-1] = ccl[0] else: @@ -476,14 +462,11 @@ def connect_with_next_base(self, bdcl): dci += len(ccl)-1 # deleted dc, added rest # END handle chunk replacement - # END for each chunk if nfc == slen: - self.frozen = True return False # END handle completeness - return True From 9b0773b92df4b4a2a53497efc9c1489028d403d7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:32:14 +0200 Subject: [PATCH 0074/3719] Removed previous non-reverse delta-application functionality. Although it was slightly faster, this new version only needs a faster slicing, which consumes ridiculous amounts of time --- fun.py | 134 ++++++++++++------------------------------------------ stream.py | 9 ++-- 2 files changed, 32 insertions(+), 111 deletions(-) diff --git a/fun.py b/fun.py index 814c22a6a..4d5edda9c 100644 --- a/fun.py +++ b/fun.py @@ -62,7 +62,6 @@ def _set_delta_rbound(d, size): # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE - return d def _move_delta_lbound(d, bytes): @@ -76,14 +75,14 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes d.ts -= bytes - if d.has_data(): + if d.data is not None: d.data = d.data[bytes:] # END handle data return d def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data, src.flags) + return DeltaChunk(src.to, src.ts, src.so, src.data) def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer @@ -92,8 +91,6 @@ def delta_chunk_apply(dc, bbuf, write): if dc.data is None: # COPY DATA FROM SOURCE write(buffer(bbuf, dc.so, dc.ts)) - elif isinstance(dc.data, DeltaChunkList): - delta_list_apply(dc.data, bbuf, write, dc.so, dc.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -105,6 +102,7 @@ def delta_chunk_apply(dc, bbuf, write): # END handle truncation # END handle chunk mode + class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -114,51 +112,25 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - 'flags' # currently only True or False ) - def __init__(self, to, ts, so, data, flags): + def __init__(self, to, ts, so, data): self.to = to self.ts = ts self.so = so self.data = data - self.flags = flags def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s, %i)" % (self.to, self.ts, self.so, self.data or "", self.flags) + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") #{ Interface - def copy_offset(self): - """:return: offset to apply when copying from a base buffer, or 0 - if this is not a copying delta chunk""" - - if self.data is not None: - if isinstance(self.data, DeltaChunkList): - return self.data.lbound() + self.so - else: - return self.so - # END handle data type - return 0 - def rbound(self): return self.to + self.ts def has_data(self): """:return: True if the instance has data to add to the target stream""" - return self.data is not None and not isinstance(self.data, DeltaChunkList) - - def has_copy_chunklist(self): - """:return: True if we copy our data from a chunklist""" - return self.data is not None and isinstance(self.data, DeltaChunkList) - - def set_copy_chunklist(self, dcl): - """Set the deltachunk list to be used as basis for copying. - :note: only works if this chunk is a copy delta chunk""" - self.data = dcl - self.so = 0 # allows lbound moves to be virtual - - + return self.data is not None #} END interface @@ -239,21 +211,20 @@ def delta_list_slice(dcl, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: DeltaChunkList (copy) which represents the given chunk""" - if len(dcl) == 0: - return DeltaChunkList() - - absofs = max(absofs, dcl.lbound()) - size = min(dcl.rbound() - dcl.lbound(), size) + dcllbound = dcl.lbound() + absofs = max(absofs, dcllbound) + size = min(dcl.rbound() - dcllbound, size) cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = dcl.__class__() + ndcl = DeltaChunkList() + lappend = ndcl.append if cd.to != absofs: tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) - ndcl.append(tcd) + lappend(tcd) size -= tcd.ts cdi += 1 # END lbound overlap handling @@ -262,12 +233,12 @@ def delta_list_slice(dcl, absofs, size): # are we larger than the current block cd = dcl[cdi] if cd.ts <= size: - ndcl.append(delta_duplicate(cd)) + lappend(delta_duplicate(cd)) size -= cd.ts else: tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) - ndcl.append(tcd) + lappend(tcd) size -= tcd.ts break # END hadle size @@ -301,19 +272,6 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl): - """Connect this instance's delta chunks virtually with the given base. - This means that all copy deltas will simply apply to the given region - of the given base. Afterwards, the base is optimized so that add-deltas - will be truncated to the region actually used, or removed completely where - adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base""" - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts)) - # END handle overlap - # END for each dc - def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines for performance""" @@ -333,7 +291,7 @@ def compress(self): while i < slen: dc = self[i] i += 1 - if not dc.has_data(): + if dc.data is None: if first_data_index is not None and i-2-first_data_index > 1: #if first_data_index is not None: nd = StringIO() # new data @@ -345,7 +303,7 @@ def compress(self): del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf, False)) + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) slen = len(self) i = first_data_index + 1 @@ -381,8 +339,6 @@ def check_integrity(self, target_size=-1): assert dc.ts > 0 if dc.has_data(): assert len(dc.data) >= dc.ts - if dc.has_copy_chunklist(): - assert dc.ts <= dc.data.size() # END for each dc left = islice(self, 0, len(self)-1) @@ -417,16 +373,8 @@ def connect_with_next_base(self, bdcl): dc = self[dci] dci += 1 - if dc.flags: - nfc += 1 - continue - # END skip frozen chunks - - # all data chunks must be frozen, we are topmost already - # (Also if its a copy operation onto the lowest base, but we cannot - # determine that without the number of deltas to come) - if dc.has_data(): - dc.flags = True + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: nfc += 1 continue # END skip add chunks @@ -448,7 +396,6 @@ def connect_with_next_base(self, bdcl): if len(ccl) == 1: self[dci-1] = ccl[0] else: - # maybe try to compute the expenses here, and pick the right algorithm # It would normally be faster than copying everything physically though # TODO: Use a deque here, and decide by the index whether to extend @@ -592,33 +539,16 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw -def reverse_connect_deltas(dcl, dstreams): +def connect_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: see 3 - :param dstreams: iterable of delta stream objects. They must be ordered latest first, - hence the delta to be applied last comes first, then its ancestors - :return: None""" - raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") - -def connect_deltas(dstreams, reverse): - """Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks - :param dstreams: iterable of delta stream objects. They must be ordered latest last, - hence the delta to be applied last comes last, its oldest ancestor first - :param reverse: If False, the given iterable of delta-streams returns - items in from latest ancestor to the last delta. - If True, deltas are ordered so that the one to be applied last comes first. + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base - tdcl = None # topmost dcl, only effective if reverse is True - - if reverse: - dcl = tdcl = TopdownDeltaChunkList() - else: - dcl = DeltaChunkList() - # END handle type of first chunk list + tdcl = None # topmost dcl + dcl = tdcl = TopdownDeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -665,12 +595,12 @@ def connect_deltas(dstreams, reverse): rbound > base_size): break - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None, False)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c], False)) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -682,12 +612,8 @@ def connect_deltas(dstreams, reverse): # merge the lists ! if bdcl is not None: - if tdcl: - if not tdcl.connect_with_next_base(dcl): - break - # END early abort - else: - dcl.connect_with(bdcl) + if not tdcl.connect_with_next_base(dcl): + break # END handle merge # prepare next base @@ -695,11 +621,7 @@ def connect_deltas(dstreams, reverse): dcl = DeltaChunkList() # END for each delta stream - if tdcl: - return tdcl - else: - return bdcl - + return tdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ diff --git a/stream.py b/stream.py index 40a0c6c6a..5292ce512 100644 --- a/stream.py +++ b/stream.py @@ -328,17 +328,16 @@ def __init__(self, stream_list): def _set_cache_(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python - # hence we apply a worst-case scenario here - # TODO: read the final size from the deltastream - have to partly unpack - # if len(self._dstreams) * self._size < self.k_max_memory_move: + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = connect_deltas(self._dstreams, reverse=True) + dcl = connect_deltas(self._dstreams) if len(dcl) == 0: self._size = 0 From 3837806673b992c1c0cb64203b50d9f1054e44db Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:45:54 +0200 Subject: [PATCH 0075/3719] Improved performance of delta_chunk_slice method a tiny bit, but it really needs to go to c --- fun.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fun.py b/fun.py index 4d5edda9c..75ff800be 100644 --- a/fun.py +++ b/fun.py @@ -210,14 +210,14 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): def delta_list_slice(dcl, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. - :return: DeltaChunkList (copy) which represents the given chunk""" + :return: list (copy) which represents the given chunk""" dcllbound = dcl.lbound() absofs = max(absofs, dcllbound) size = min(dcl.rbound() - dcllbound, size) cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = DeltaChunkList() + ndcl = list() lappend = ndcl.append if cd.to != absofs: From 89408f8ec351c1d14f66caf53a2c1e163bde0f27 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 23:08:57 +0200 Subject: [PATCH 0076/3719] Initial frame of the connect_delta method, which seems to do something. Debugging is hellish, you really have to use python exception to get information out of there, printf doesn't do anything for some reason --- Makefile | 24 +++++ _fun.c | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++++- fun.py | 5 + stream.py | 5 +- 4 files changed, 310 insertions(+), 3 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..190a66b03 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +PYTHON = python +SETUP = $(PYTHON) setup.py +TESTRUNNER = $(shell which nosetests) +TESTFLAGS = + +all: build + +doc:: + make -C doc/ html + +build:: + $(SETUP) build + $(SETUP) build_ext -i + +install:: + $(SETUP) install + +clean:: + $(SETUP) clean --all + rm -f *.so + +coverage:: build + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=dulwich --with-coverage --cover-erase --cover-inclusive gitdb + diff --git a/_fun.c b/_fun.c index ce9f25b16..e9f769daa 100644 --- a/_fun.c +++ b/_fun.c @@ -1,5 +1,7 @@ #include #include +#include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -82,16 +84,289 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) } +typedef unsigned long long ull; + +// Internal Delta Chunk Objects +typedef struct { + ull to; + ull ts; + ull so; + PyObject* data; + + void* next; +} DeltaChunk; + + +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data, DeltaChunk* next) +{ + dc->to = to; + dc->ts = ts; + dc->so = so; + Py_XINCREF(data); + dc->data = data; + + dc->next = next; +} + +void DC_destroy(DeltaChunk* dc) +{ + Py_XDECREF(dc->data); +} + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunk* head; + DeltaChunk* tail; + ull size; + +} DeltaChunkList; + +ull DC_rbound(DeltaChunk* dc) +{ + return dc->to + dc->ts; +} + + +static +int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) +{ + ((DeltaChunkList*)self)->head = NULL; + return 1; +} + +static +void DCL_dealloc(DeltaChunkList* self) +{ + // TODO: deallocate linked list + if (self->head){ + self->head = NULL; + self->tail = NULL; + self->size = 0; + } +} + +static +PyObject* DCL_len(PyObject* self) +{ + return PyLong_FromUnsignedLongLong(0); +} + +static +PyObject* DCL_rbound(DeltaChunkList* self) +{ + if (!self->head) + return PyLong_FromUnsignedLongLong(0); + return PyLong_FromUnsignedLongLong(DC_rbound(self->tail)); +} + +static +PyObject* DCL_apply(PyObject* self, PyObject* args) +{ + + Py_RETURN_NONE; +} + + + +static PyMethodDef DCL_methods[] = { + {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, + {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_rbound, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject DeltaChunkListType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "DeltaChunkList", /*tp_name*/ + sizeof(DeltaChunkList), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)DCL_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "Minimal Delta Chunk List",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DCL_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)DCL_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static inline +ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ + ull size = 0; + Py_ssize_t i = 0; + const char* dend = data + dlen; + for (data = data + offset; data < dend; data+=1, i+=1){ + char c = *data; + size |= (c & 0x7f) << i*7; + if (!(c & 0x80)){ + break; + } + }// END while in range + + *out_bytes_read = i+offset; + return size; +} + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +{ + // obtain iterator + PyObject* stream_iter = 0; + if (!PyIter_Check(dstreams)){ + stream_iter = PyObject_GetIter(dstreams); + if (!stream_iter){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); + return NULL; + } + } else { + stream_iter = dstreams; + } + + DeltaChunkList* bdcl = 0; + DeltaChunkList* tdcl = 0; + DeltaChunkList* dcl = 0; + + dcl = tdcl = PyObject_New(DeltaChunkList, &DeltaChunkListType); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + return NULL; + } + + unsigned int dsi; + PyObject* ds; + int error = 0; + for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + { + PyObject* db = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(db)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); + goto loop_end; + } + + const char* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + + // read header + Py_ssize_t ofs = 0; + const ull base_size = msb_size(data, dlen, 0, &ofs); + const ull target_size = msb_size(data, dlen, ofs, &ofs); + + // parse command stream + const char* dend = data + dlen; + ull tbw = 0; // Amount of target bytes written + for (data = data + ofs; data < dend; ++data) + { + const char cmd = *data; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + const unsigned long rbound = cp_off + cp_size; + if (rbound < cp_size || + rbound > base_size){ + goto loop_end; + } + + // TODO: Add node + tbw += cp_size; + + } else if (cmd) { + // TODO: Add node + tbw += cmd; + } else { + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + goto loop_end; + } + }// END handle command opcodes + + assert(tbw == target_size); + +loop_end: + // perform cleanup + Py_DECREF(ds); + Py_DECREF(db); + + if (error){ + break; + } + }// END for each stream object + + if (dsi == 0 && ! error){ + PyErr_SetString(PyExc_ValueError, "No streams provided"); + } + + if (stream_iter != dstreams){ + Py_DECREF(stream_iter); + } + + if (error){ + return NULL; + } + + return (PyObject*)tdcl; +} + static PyMethodDef py_fun[] = { - { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL }, + { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; -void init_fun(void) +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC init_fun(void) { PyObject *m; + DeltaChunkListType.tp_new = PyType_GenericNew; + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + m = Py_InitModule3("_fun", py_fun, NULL); if (m == NULL) return; + + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "Noddy", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index 75ff800be..a4da30903 100644 --- a/fun.py +++ b/fun.py @@ -701,3 +701,8 @@ def is_equal_canonical_sha(canonical_length, match, sha1): #} END routines + +try: + from _fun import connect_deltas +except ImportError: + pass diff --git a/stream.py b/stream.py index 5292ce512..169104625 100644 --- a/stream.py +++ b/stream.py @@ -338,8 +338,11 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = connect_deltas(self._dstreams) + assert dcl is not None - if len(dcl) == 0: + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.__len__() == 0: self._size = 0 self._mm_target = allocate_memory(0) return From 2409fafca6f0518500571bcf82e92554f2c50b85 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 09:19:12 +0200 Subject: [PATCH 0077/3719] Implemented a few more functions, but I realize the vector implementation actually wants to be in a separate structure --- _fun.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/_fun.c b/_fun.c index e9f769daa..8abc5bcbb 100644 --- a/_fun.c +++ b/_fun.c @@ -116,9 +116,9 @@ void DC_destroy(DeltaChunk* dc) typedef struct { PyObject_HEAD // ----------- - DeltaChunk* head; - DeltaChunk* tail; + DeltaChunk* mem; ull size; + ull reserved_size; } DeltaChunkList; @@ -127,22 +127,65 @@ ull DC_rbound(DeltaChunk* dc) return dc->to + dc->ts; } +static +int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) +{ + self->mem = NULL; // Memory + self->size = 0; // Size in DeltaChunks + self->reserved_size = 0; // Reserve in DeltaChunks + return 1; +} + +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +static +int DCL_grow(DeltaChunkList* self, ull num_dc) +{ + const ull grow_by_chunks = (self->size + num_dc) - self->reserved_size; + if (grow_by_chunks <= 0){ + return 1; + } + + if (self->mem){ + self->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + } else { + self->mem = PyMem_Realloc(self->mem, (self->size + grow_by_chunks)*sizeof(DeltaChunk)); + } + + return self->mem != NULL; +} + static int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) { - ((DeltaChunkList*)self)->head = NULL; - return 1; + if(PySequence_Size(args) > 1){ + PyErr_SetString(PyExc_ValueError, "Zero or one arguments are allowed, providing the initial size of the queue in DeltaChunks"); + return 0; + } + + ull init_size = 0; + PyArg_ParseTuple(args, "K", &init_size); + if (init_size == 0){ + init_size = 125000; + } + + return DCL_grow(self, init_size); } static void DCL_dealloc(DeltaChunkList* self) { // TODO: deallocate linked list - if (self->head){ - self->head = NULL; - self->tail = NULL; + if (self->mem){ + PyMem_Free(self->mem); self->size = 0; + self->reserved_size = 0; + self->mem = 0; } } @@ -153,11 +196,18 @@ PyObject* DCL_len(PyObject* self) } static -PyObject* DCL_rbound(DeltaChunkList* self) +inline +ull DCL_rbound(DeltaChunkList* self) +{ + if (!self->mem | !self->size) + return 0; + return DC_rbound(&(self->mem[self->size-1])); +} + +static +PyObject* DCL_py_rbound(DeltaChunkList* self) { - if (!self->head) - return PyLong_FromUnsignedLongLong(0); - return PyLong_FromUnsignedLongLong(DC_rbound(self->tail)); + return PyLong_FromUnsignedLongLong(DCL_rbound(self)); } static @@ -172,7 +222,7 @@ PyObject* DCL_apply(PyObject* self, PyObject* args) static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, - {"rbound", (PyCFunction)DCL_rbound, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -215,7 +265,7 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_dictoffset */ (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + (newfunc)DCL_new, /* tp_new */ }; @@ -233,6 +283,7 @@ ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* o }// END while in range *out_bytes_read = i+offset; + assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull)); return size; } From 511a29dab7c7a507abc3bc656b5e9f5ab2db85a3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 09:54:15 +0200 Subject: [PATCH 0078/3719] DeltaChunkVector is now a separate structure. I wished I had c++, but ... its probably a good exercise --- _fun.c | 126 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/_fun.c b/_fun.c index 8abc5bcbb..fb83f26a6 100644 --- a/_fun.c +++ b/_fun.c @@ -85,27 +85,26 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; +typedef unsigned int uint; + +// DELTA CHUNK +//////////////// // Internal Delta Chunk Objects typedef struct { ull to; ull ts; ull so; PyObject* data; - - void* next; } DeltaChunk; - -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data, DeltaChunk* next) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data) { dc->to = to; dc->ts = ts; dc->so = so; Py_XINCREF(data); dc->data = data; - - dc->next = next; } void DC_destroy(DeltaChunk* dc) @@ -113,28 +112,20 @@ void DC_destroy(DeltaChunk* dc) Py_XDECREF(dc->data); } -typedef struct { - PyObject_HEAD - // ----------- - DeltaChunk* mem; - ull size; - ull reserved_size; - -} DeltaChunkList; - ull DC_rbound(DeltaChunk* dc) { return dc->to + dc->ts; } -static -int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) -{ - self->mem = NULL; // Memory - self->size = 0; // Size in DeltaChunks - self->reserved_size = 0; // Reserve in DeltaChunks - return 1; -} + +// DELTA CHUNK VECTOR +///////////////////// + +typedef struct { + DeltaChunk* mem; // Memory + Py_ssize_t size; // Size in DeltaChunks + Py_ssize_t reserved_size; // Reserve in DeltaChunks +} DeltaChunkVector; /* Grow the delta chunk list by the given amount of bytes. @@ -143,20 +134,76 @@ large enough. Return 1 on success, 0 on failure */ static -int DCL_grow(DeltaChunkList* self, ull num_dc) +int DCV_grow(DeltaChunkVector* vec, uint num_dc) { - const ull grow_by_chunks = (self->size + num_dc) - self->reserved_size; + const ull grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; if (grow_by_chunks <= 0){ return 1; } - if (self->mem){ - self->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + if (vec->mem){ + vec->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); } else { - self->mem = PyMem_Realloc(self->mem, (self->size + grow_by_chunks)*sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, (vec->size + grow_by_chunks)*sizeof(DeltaChunk)); } - return self->mem != NULL; + return vec->mem != NULL; +} + +int DCV_init(DeltaChunkVector* vec, ull initial_size) +{ + vec->mem = NULL; + vec->size = 0; + vec->reserved_size = 0; + + return DCV_grow(vec, initial_size); +} + + +void DCV_dealloc(DeltaChunkVector* vec) +{ + if (vec->mem){ + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +static inline +ull DCV_len(DeltaChunkVector* vec) +{ + return vec->size; +} + +// Return item at index +static inline +DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) +{ + assert(i < vec->size && vec->mem); + return &(vec->mem[i]); +} + +static inline +int DCV_empty(DeltaChunkVector* vec) +{ + return vec->size == 0; +} + +// DELTA CHUNK LIST (PYTHON) +///////////////////////////// + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunkVector vec; + +} DeltaChunkList; + +static +int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) +{ + return DCV_init(&self->vec, 0); } @@ -174,34 +221,27 @@ int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) init_size = 125000; } - return DCL_grow(self, init_size); + return DCV_grow(&self->vec, init_size); } static void DCL_dealloc(DeltaChunkList* self) { - // TODO: deallocate linked list - if (self->mem){ - PyMem_Free(self->mem); - self->size = 0; - self->reserved_size = 0; - self->mem = 0; - } + DCV_dealloc(&self->vec); } static -PyObject* DCL_len(PyObject* self) +PyObject* DCL_len(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(0); + return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); } -static -inline +static inline ull DCL_rbound(DeltaChunkList* self) { - if (!self->mem | !self->size) + if (DCV_empty(&self->vec)) return 0; - return DC_rbound(&(self->mem[self->size-1])); + return DC_rbound(DCV_get(&self->vec, self->vec.size - 1)); } static From f030fa1d7ef6a43cc05b11e3be2108f83d9683f9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 11:25:23 +0200 Subject: [PATCH 0079/3719] Weird bug causes crash, its memory related of course. GDB tells me where, but the why is still a mystery --- _fun.c | 82 +++++++++++++++++++++++++++++++++++++------------------ stream.py | 3 ++ 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/_fun.c b/_fun.c index fb83f26a6..47f5d8ecf 100644 --- a/_fun.c +++ b/_fun.c @@ -136,16 +136,18 @@ Return 1 on success, 0 on failure static int DCV_grow(DeltaChunkVector* vec, uint num_dc) { - const ull grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; + const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; if (grow_by_chunks <= 0){ return 1; } - if (vec->mem){ - vec->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + if (vec->mem == NULL){ + vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(vec->mem)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->size + grow_by_chunks)*sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(vec->mem)); } + assert(vec->mem != NULL); + vec->reserved_size = vec->reserved_size + grow_by_chunks; return vec->mem != NULL; } @@ -159,17 +161,6 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) return DCV_grow(vec, initial_size); } - -void DCV_dealloc(DeltaChunkVector* vec) -{ - if (vec->mem){ - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - static inline ull DCV_len(DeltaChunkVector* vec) { @@ -181,7 +172,7 @@ static inline DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); - return &(vec->mem[i]); + return &vec->mem[i]; } static inline @@ -190,6 +181,48 @@ int DCV_empty(DeltaChunkVector* vec) return vec->size == 0; } +// Return end pointer of the vector +static inline +DeltaChunk* DCV_end(DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return &vec->mem[vec->size]; +} + +void DCV_dealloc(DeltaChunkVector* vec) +{ + if (vec->mem){ + if (vec->size){ + const DeltaChunk* end = DCV_end(vec); + DeltaChunk* i; + for(i = &vec->mem[0]; i < end; i++){ + DC_destroy(i); + } + } + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +// Append num-chunks to the end of the list, possibly reallocating existing ones +// Return a pointer to the first of the added items. They are not yet initialized +// If num-chunks == 0, it returns the end pointer of the allocated memory +static inline +DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) +{ + if (vec->size + num_chunks > vec->reserved_size){ + if (!DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size)){ + Py_FatalError("Could not allocate memory for append operation"); + } + } + Py_FatalError("Could not allocate memory for append operation"); + Py_ssize_t old_size = vec->size; + vec->size += num_chunks; + return &vec->mem[old_size]; +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -200,12 +233,6 @@ typedef struct { } DeltaChunkList; -static -int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) -{ - return DCV_init(&self->vec, 0); -} - static int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) @@ -215,19 +242,20 @@ int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) return 0; } + assert(self->vec.mem == NULL); + ull init_size = 0; PyArg_ParseTuple(args, "K", &init_size); if (init_size == 0){ - init_size = 125000; + init_size = 12500; } - - return DCV_grow(&self->vec, init_size); + return DCV_init(&self->vec, init_size); } static void DCL_dealloc(DeltaChunkList* self) { - DCV_dealloc(&self->vec); + DCV_dealloc(&(self->vec)); } static @@ -305,7 +333,7 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_dictoffset */ (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - (newfunc)DCL_new, /* tp_new */ + 0, /* tp_new */ }; diff --git a/stream.py b/stream.py index 169104625..6d7479d1c 100644 --- a/stream.py +++ b/stream.py @@ -339,6 +339,9 @@ def _set_cache_(self, attr): # the final delta data stream. dcl = connect_deltas(self._dstreams) assert dcl is not None + print "got dcl" + del(dcl) + print "dealloc worked" # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 445b71637576649fd6f1f3c287f50eec24d4fbf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 12:43:57 +0200 Subject: [PATCH 0080/3719] Wow, this was a lesson. My full hatred goes to python, and C, and everything ... cool if you control everything, but not cool if an Object_New call doesn't do anything for you - creating a new instance of an own type in python doesn't appear to be that easy after all, at least not if you want your initializers/new procs to be called --- _fun.c | 60 +++++++++++++++++++++++++++++++------------------------ stream.py | 3 --- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/_fun.c b/_fun.c index 47f5d8ecf..3a1bd0355 100644 --- a/_fun.c +++ b/_fun.c @@ -192,13 +192,12 @@ DeltaChunk* DCV_end(DeltaChunkVector* vec) void DCV_dealloc(DeltaChunkVector* vec) { if (vec->mem){ - if (vec->size){ - const DeltaChunk* end = DCV_end(vec); - DeltaChunk* i; - for(i = &vec->mem[0]; i < end; i++){ - DC_destroy(i); - } + const DeltaChunk* end = DCV_end(vec); + DeltaChunk* i; + for(i = vec->mem; i < end; i++){ + DC_destroy(i); } + PyMem_Free(vec->mem); vec->size = 0; vec->reserved_size = 0; @@ -207,7 +206,7 @@ void DCV_dealloc(DeltaChunkVector* vec) } // Append num-chunks to the end of the list, possibly reallocating existing ones -// Return a pointer to the first of the added items. They are not yet initialized +// Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) @@ -220,6 +219,11 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; vec->size += num_chunks; + + for(;old_size < vec->size; ++old_size){ + DC_init(DCV_get(vec, old_size), 0, 0, 0, NULL); + } + return &vec->mem[old_size]; } @@ -235,21 +239,15 @@ typedef struct { static -int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) +int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) { - if(PySequence_Size(args) > 1){ - PyErr_SetString(PyExc_ValueError, "Zero or one arguments are allowed, providing the initial size of the queue in DeltaChunks"); - return 0; + if(args && PySequence_Size(args) > 0){ + PyErr_SetString(PyExc_ValueError, "Too many arguments"); + return -1; } - assert(self->vec.mem == NULL); - - ull init_size = 0; - PyArg_ParseTuple(args, "K", &init_size); - if (init_size == 0){ - init_size = 12500; - } - return DCV_init(&self->vec, init_size); + DCV_init(&self->vec, 0); + return 0; } static @@ -285,8 +283,6 @@ PyObject* DCL_apply(PyObject* self, PyObject* args) Py_RETURN_NONE; } - - static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, @@ -331,12 +327,24 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ + (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + 0, /* tp_new */ }; +// Makes a new copy of the DeltaChunkList - you have to do everything yourselve +// in C ... want C++ !! +DeltaChunkList* DCL_new_instance(void) +{ + DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); + assert(dcl); + + DCL_init(dcl, 0, 0); + assert(dcl->vec.size == 0); + return dcl; +} + static inline ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ ull size = 0; @@ -351,7 +359,7 @@ ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* o }// END while in range *out_bytes_read = i+offset; - assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull)); + assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull) * 8); return size; } @@ -373,7 +381,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkList* tdcl = 0; DeltaChunkList* dcl = 0; - dcl = tdcl = PyObject_New(DeltaChunkList, &DeltaChunkListType); + dcl = tdcl = DCL_new_instance(); + assert(dcl != NULL); if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); return NULL; @@ -478,7 +487,6 @@ PyMODINIT_FUNC init_fun(void) { PyObject *m; - DeltaChunkListType.tp_new = PyType_GenericNew; if (PyType_Ready(&DeltaChunkListType) < 0) return; diff --git a/stream.py b/stream.py index 6d7479d1c..169104625 100644 --- a/stream.py +++ b/stream.py @@ -339,9 +339,6 @@ def _set_cache_(self, attr): # the final delta data stream. dcl = connect_deltas(self._dstreams) assert dcl is not None - print "got dcl" - del(dcl) - print "dealloc worked" # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 9ba93c0deb4f46b524e28c8103b35f408a936e04 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 14:58:38 +0200 Subject: [PATCH 0081/3719] Apparently, the most serious memory bugs are fixed for now, lets get back to the actual thing --- _fun.c | 73 +++++++++++++++++++++++++++++++++++++++++++------------ stream.py | 1 - 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/_fun.c b/_fun.c index 3a1bd0355..7089cfe3d 100644 --- a/_fun.c +++ b/_fun.c @@ -2,6 +2,7 @@ #include #include #include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -87,7 +88,6 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; - // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -142,13 +142,17 @@ int DCV_grow(DeltaChunkVector* vec, uint num_dc) } if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(vec->mem)); + vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(DeltaChunk)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(vec->mem)); + vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); } assert(vec->mem != NULL); vec->reserved_size = vec->reserved_size + grow_by_chunks; +#ifdef DEBUG + fprintf(stderr, "Allocated %i bytes at %p, to hold up to %i chunks\n", (int)((vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)), vec->mem, (int)(vec->reserved_size + grow_by_chunks)); +#endif + return vec->mem != NULL; } @@ -192,7 +196,11 @@ DeltaChunk* DCV_end(DeltaChunkVector* vec) void DCV_dealloc(DeltaChunkVector* vec) { if (vec->mem){ - const DeltaChunk* end = DCV_end(vec); +#ifdef DEBUG + fprintf(stderr, "Freeing %p\n", (void*)vec->mem); +#endif + + const DeltaChunk* end = &vec->mem[vec->size]; DeltaChunk* i; for(i = vec->mem; i < end; i++){ DC_destroy(i); @@ -342,6 +350,7 @@ DeltaChunkList* DCL_new_instance(void) DCL_init(dcl, 0, 0); assert(dcl->vec.size == 0); + assert(dcl->vec.mem == NULL); return dcl; } @@ -377,16 +386,13 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkList* bdcl = 0; - DeltaChunkList* tdcl = 0; - DeltaChunkList* dcl = 0; + DeltaChunkVector bdcv; + DeltaChunkVector tdcv; + DeltaChunkVector dcv; + DCV_init(&bdcv, 0); + DCV_init(&dcv, 0); + DCV_init(&tdcv, 0); - dcl = tdcl = DCL_new_instance(); - assert(dcl != NULL); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - return NULL; - } unsigned int dsi; PyObject* ds; @@ -408,6 +414,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_ssize_t ofs = 0; const ull base_size = msb_size(data, dlen, 0, &ofs); const ull target_size = msb_size(data, dlen, ofs, &ofs); + + // estimate number of ops - assume one third adds, half two byte (size+offset) copies + const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + DCV_grow(&dcv, approx_num_cmds); // parse command stream const char* dend = data + dlen; @@ -431,7 +441,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ - goto loop_end; + break; } // TODO: Add node @@ -446,8 +456,19 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } }// END handle command opcodes - assert(tbw == target_size); + + // swap the vector + // Skip the first vector, as it is also used as top chunk vector + if (bdcv.mem != tdcv.mem){ + DCV_dealloc(&bdcv); + } + bdcv = dcv; + if (dsi == 0){ + tdcv = dcv; + } + DCV_init(&dcv, 0); + loop_end: // perform cleanup @@ -467,11 +488,31 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } + DCV_dealloc(&bdcv); + if (dsi > 1){ + // otherwise dcv equals tcl + DCV_dealloc(&dcv); + } + + // Return the actual python object - its just a container + DeltaChunkList* dcl = DCL_new_instance(); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + // Otherwise tdcv would be deallocated by the chunk list + DCV_dealloc(&tdcv); + error = 1; + } else { + // Plain copy, don't deallocate + dcl->vec = tdcv; + } + if (error){ + // Will dealloc tdcv + Py_XDECREF(dcl); return NULL; } - return (PyObject*)tdcl; + return (PyObject*)dcl; } static PyMethodDef py_fun[] = { diff --git a/stream.py b/stream.py index 169104625..af4591f85 100644 --- a/stream.py +++ b/stream.py @@ -338,7 +338,6 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = connect_deltas(self._dstreams) - assert dcl is not None # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 489f763308d4982a5220a648b92ecb2cc82f4ae4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 16:04:55 +0200 Subject: [PATCH 0082/3719] Now adding chunks to the vectors, next up is to implement the actual chunk merging --- _fun.c | 104 ++++++++++++++++++++++++++++++++++++++------------------- fun.py | 1 + 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/_fun.c b/_fun.c index 7089cfe3d..2ad8d641d 100644 --- a/_fun.c +++ b/_fun.c @@ -3,6 +3,7 @@ #include #include #include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -87,6 +88,7 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; +typedef unsigned char uchar; // DELTA CHUNK //////////////// @@ -95,21 +97,38 @@ typedef struct { ull to; ull ts; ull so; - PyObject* data; + uchar* data; } DeltaChunk; -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) { dc->to = to; dc->ts = ts; dc->so = so; - Py_XINCREF(data); - dc->data = data; + dc->data = NULL; } void DC_destroy(DeltaChunk* dc) { - Py_XDECREF(dc->data); + if (dc->data){ + PyMem_Free((void*)dc->data); + } +} + +// Store a copy of data in our instance +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen) +{ + if (dc->data){ + PyMem_Free((void*)dc->data); + } + + if (data == 0){ + dc->data = NULL; + return; + } + + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy(dc->data, data, dlen); } ull DC_rbound(DeltaChunk* dc) @@ -146,7 +165,11 @@ int DCV_grow(DeltaChunkVector* vec, uint num_dc) } else { vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); } - assert(vec->mem != NULL); + + if (vec->mem == NULL){ + Py_FatalError("Could not allocate memory for append operation"); + } + vec->reserved_size = vec->reserved_size + grow_by_chunks; #ifdef DEBUG @@ -220,21 +243,33 @@ static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) { if (vec->size + num_chunks > vec->reserved_size){ - if (!DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size)){ - Py_FatalError("Could not allocate memory for append operation"); - } + DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size); } Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; vec->size += num_chunks; for(;old_size < vec->size; ++old_size){ - DC_init(DCV_get(vec, old_size), 0, 0, 0, NULL); + DC_init(DCV_get(vec, old_size), 0, 0, 0); } return &vec->mem[old_size]; } +// Append one chunk to the end of the list, and return a pointer to it +// It will have been initialized. +static inline +DeltaChunk* DCV_append(DeltaChunkVector* vec) +{ + if (vec->size + 1 > vec->reserved_size){ + DCV_grow(vec, 1); + } + + DeltaChunk* next = vec->mem + vec->size; + vec->size += 1; + return next; +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -354,21 +389,18 @@ DeltaChunkList* DCL_new_instance(void) return dcl; } -static inline -ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ - ull size = 0; - Py_ssize_t i = 0; - const char* dend = data + dlen; - for (data = data + offset; data < dend; data+=1, i+=1){ - char c = *data; - size |= (c & 0x7f) << i*7; - if (!(c & 0x80)){ - break; - } - }// END while in range - - *out_bytes_read = i+offset; - assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull) * 8); +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; return size; } @@ -406,25 +438,25 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } - const char* data; + const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + const uchar* dend = data + dlen; // read header - Py_ssize_t ofs = 0; - const ull base_size = msb_size(data, dlen, 0, &ofs); - const ull target_size = msb_size(data, dlen, ofs, &ofs); + const ull base_size = msb_size(&data, dend); + const ull target_size = msb_size(&data, dend); // estimate number of ops - assume one third adds, half two byte (size+offset) copies const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); DCV_grow(&dcv, approx_num_cmds); // parse command stream - const char* dend = data + dlen; ull tbw = 0; // Amount of target bytes written - for (data = data + ofs; data < dend; ++data) + assert(data < dend); + while (data < dend) { - const char cmd = *data; + const char cmd = *data++; if (cmd & 0x80) { @@ -444,12 +476,16 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) break; } - // TODO: Add node + DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); tbw += cp_size; } else if (cmd) { - // TODO: Add node + // TODO: Compress nodes by parsing them in advance + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, cmd, 0); + DC_set_data(dc, data, cmd); tbw += cmd; + data += cmd; } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); diff --git a/fun.py b/fun.py index a4da30903..e6262b4bc 100644 --- a/fun.py +++ b/fun.py @@ -703,6 +703,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: + # raise ImportError; # DEBUG from _fun import connect_deltas except ImportError: pass From a93363cffb225520869d737de1081c1ff77ed108 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 17:45:42 +0200 Subject: [PATCH 0083/3719] prepared the slicing, as well as a few accompanying methods. There is still quite a lot functionality missing --- _fun.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 166 insertions(+), 28 deletions(-) diff --git a/_fun.c b/_fun.c index 2ad8d641d..1186c6d11 100644 --- a/_fun.c +++ b/_fun.c @@ -89,6 +89,7 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; +typedef uchar bool; // DELTA CHUNK //////////////// @@ -97,45 +98,96 @@ typedef struct { ull to; ull ts; ull so; - uchar* data; + const uchar* data; + bool data_shared; } DeltaChunk; +inline void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) { dc->to = to; dc->ts = ts; dc->so = so; dc->data = NULL; + dc->data_shared = 0; } -void DC_destroy(DeltaChunk* dc) +inline +void DC_deallocate_data(DeltaChunk* dc) { - if (dc->data){ + if (!dc->data_shared && dc->data){ PyMem_Free((void*)dc->data); } + dc->data = NULL; } -// Store a copy of data in our instance -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen) +inline +void DC_destroy(DeltaChunk* dc) { - if (dc->data){ - PyMem_Free((void*)dc->data); - } + DC_deallocate_data(dc); +} + +// Store a copy of data in our instance. If shared is 1, the data will be shared, +// hence it will only be stored, but the memory will not be touched, or copied. +inline +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) +{ + DC_deallocate_data(dc); if (data == 0){ dc->data = NULL; + dc->data_shared = 0; return; } - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy(dc->data, data, dlen); + dc->data_shared = shared; + if (shared){ + dc->data = data; + } else { + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy((void*)dc->data, (void*)data, dlen); + } + } +inline ull DC_rbound(DeltaChunk* dc) { return dc->to + dc->ts; } +// Copy all data from src to dest, the data pointer will be copied too +inline +void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) +{ + dest->to = src->to; + dest->ts = src->ts; + dest->so = src->so; + dest->data_shared = 0; + + DC_set_data(dest, src->data, src->ts, 0); +} + +// Copy all data with the given offset and size. The source offset, as well +// as the data will be truncated accordingly +inline +void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +{ + assert(size <= src->ts); + assert(src->to + ofs + size <= DC_rbound(src)); + + dest->to = src->to + ofs; + dest->ts = size; + dest->so = src->so + ofs; + + if (src->data){ + DC_set_data(dest, src->data + ofs, size, 0); + } else { + dest->data = NULL; + dest->data_shared = 0; + } +} + // DELTA CHUNK VECTOR ///////////////////// @@ -152,7 +204,7 @@ This may trigger a realloc, but will do nothing if the reserved size is already large enough. Return 1 on success, 0 on failure */ -static +inline int DCV_grow(DeltaChunkVector* vec, uint num_dc) { const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; @@ -188,35 +240,48 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) return DCV_grow(vec, initial_size); } -static inline +inline ull DCV_len(DeltaChunkVector* vec) { return vec->size; } +inline +ull DCV_lbound(DeltaChunkVector* vec) +{ + assert(vec->size && vec->mem); + return vec->mem->to; +} + // Return item at index -static inline +inline DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; } -static inline +inline +ull DCV_rbound(DeltaChunkVector* vec) +{ + return DC_rbound(DCV_get(vec, vec->size-1)); +} + +inline int DCV_empty(DeltaChunkVector* vec) { return vec->size == 0; } // Return end pointer of the vector -static inline +inline DeltaChunk* DCV_end(DeltaChunkVector* vec) { assert(!DCV_empty(vec)); return &vec->mem[vec->size]; } -void DCV_dealloc(DeltaChunkVector* vec) +void DCV_destroy(DeltaChunkVector* vec) { if (vec->mem){ #ifdef DEBUG @@ -236,6 +301,14 @@ void DCV_dealloc(DeltaChunkVector* vec) } } +// Reset this vector so that its existing memory can be filled again. +// Memory will be kept, but not cleaned up +inline +void DCV_forget_members(DeltaChunkVector* vec) +{ + vec->size = 0; +} + // Append num-chunks to the end of the list, possibly reallocating existing ones // Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory @@ -249,15 +322,17 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_ssize_t old_size = vec->size; vec->size += num_chunks; +#ifdef DEBUG for(;old_size < vec->size; ++old_size){ DC_init(DCV_get(vec, old_size), 0, 0, 0); } +#endif return &vec->mem[old_size]; } // Append one chunk to the end of the list, and return a pointer to it -// It will have been initialized. +// It will not have been initialized ! static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { @@ -270,6 +345,59 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) return next; } +// Write a slice as defined by its absolute offset in bytes and its size into the given +// destination. The individual chunks written will be a deep copy of the source +// data chunks +// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort +// of append-only memory pool would improve performance +inline +void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +{ + +} + + +// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost +// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the +// caller +static +void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +{ + DeltaChunk* dc = tdcv->mem; + DeltaChunk* end = tdcv->mem + tdcv->size; + assert(dc); + + for (;dc < end; dc++) + { + // Data chunks don't need processing + if (dc->data){ + continue; + } + + // Copy Chunk Handling + DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); + // assert(tmpl->size); + + // move target bounds + DeltaChunk* cdc = tmpl->mem; + DeltaChunk* cdcend = tmpl->mem + tmpl->size; + const ull ofs = dc->to - dc->so; + for(;cdc < cdcend; cdc++){ + cdc->to += ofs; + } + + // insert slice into our list, replacing our current chunk + if (tmpl->size == 1){ + *dc = *DCV_get(tmpl, 0); + } else { + + } + + // make sure the members will not be deallocated by the list + DCV_forget_members(tmpl); + } +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -296,7 +424,7 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) static void DCL_dealloc(DeltaChunkList* self) { - DCV_dealloc(&(self->vec)); + DCV_destroy(&(self->vec)); } static @@ -310,7 +438,7 @@ ull DCL_rbound(DeltaChunkList* self) { if (DCV_empty(&self->vec)) return 0; - return DC_rbound(DCV_get(&self->vec, self->vec.size - 1)); + return DCV_rbound(&self->vec); } static @@ -421,10 +549,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkVector bdcv; DeltaChunkVector tdcv; DeltaChunkVector dcv; + DeltaChunkVector tmpl; DCV_init(&bdcv, 0); DCV_init(&dcv, 0); DCV_init(&tdcv, 0); - + DCV_init(&tmpl, 200); unsigned int dsi; PyObject* ds; @@ -453,6 +582,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // parse command stream ull tbw = 0; // Amount of target bytes written + bool shared_data = dsi != 0; + bool is_first_run = dsi == 0; + assert(data < dend); while (data < dend) { @@ -481,9 +613,12 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } else if (cmd) { // TODO: Compress nodes by parsing them in advance + // NOTE: Compression only necessary for all other deltas, not + // for the first one, as we will share the data. It really depends + // What's faster DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd); + DC_set_data(dc, data, cmd, shared_data); tbw += cmd; data += cmd; } else { @@ -493,18 +628,20 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } }// END handle command opcodes assert(tbw == target_size); - + + if (!is_first_run){ + DCV_connect_with_base(&tdcv, &dcv, &tmpl); + } // swap the vector // Skip the first vector, as it is also used as top chunk vector if (bdcv.mem != tdcv.mem){ - DCV_dealloc(&bdcv); + DCV_destroy(&bdcv); } bdcv = dcv; - if (dsi == 0){ + if (is_first_run){ tdcv = dcv; } DCV_init(&dcv, 0); - loop_end: // perform cleanup @@ -524,10 +661,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - DCV_dealloc(&bdcv); + DCV_destroy(&tmpl); + DCV_destroy(&bdcv); if (dsi > 1){ // otherwise dcv equals tcl - DCV_dealloc(&dcv); + DCV_destroy(&dcv); } // Return the actual python object - its just a container @@ -535,7 +673,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); // Otherwise tdcv would be deallocated by the chunk list - DCV_dealloc(&tdcv); + DCV_destroy(&tdcv); error = 1; } else { // Plain copy, don't deallocate From 166e538f9aab8db7ab30b6e8b3be407200a7e3c1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 18:17:19 +0200 Subject: [PATCH 0084/3719] Enhanced memory handling within the delta-stream parsing method. Removed the base delta chunk vector, which was a reminder of old (python) times which are long gone --- _fun.c | 88 +++++++++++++++++++++++++++++++++++++++------------------- fun.py | 4 +-- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/_fun.c b/_fun.c index 1186c6d11..142795e6c 100644 --- a/_fun.c +++ b/_fun.c @@ -198,46 +198,62 @@ typedef struct { Py_ssize_t reserved_size; // Reserve in DeltaChunks } DeltaChunkVector; -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ + + +// Reserve enough memory to hold the given amount of delta chunks +// Return 1 on success inline -int DCV_grow(DeltaChunkVector* vec, uint num_dc) +int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) { - const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; - if (grow_by_chunks <= 0){ + if (num_dc <= vec->reserved_size){ return 1; } +#ifdef DEBUG + bool was_null = vec->mem == NULL; +#endif + if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(DeltaChunk)); + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); } if (vec->mem == NULL){ Py_FatalError("Could not allocate memory for append operation"); } - vec->reserved_size = vec->reserved_size + grow_by_chunks; + vec->reserved_size = num_dc; #ifdef DEBUG - fprintf(stderr, "Allocated %i bytes at %p, to hold up to %i chunks\n", (int)((vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)), vec->mem, (int)(vec->reserved_size + grow_by_chunks)); + const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; + if (!was_null) + format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); #endif return vec->mem != NULL; } +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +inline +int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +{ + return DCV_reserve_memory(vec, vec->reserved_size + num_dc); +} + int DCV_init(DeltaChunkVector* vec, ull initial_size) { vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; - return DCV_grow(vec, initial_size); + return DCV_grow_by(vec, initial_size); } inline @@ -309,6 +325,24 @@ void DCV_forget_members(DeltaChunkVector* vec) vec->size = 0; } +// Reset the vector so that its size will be zero, and its members will +// have been deallocated properly. +// It will keep its memory though, and hence can be filled again +inline +void DCV_reset(DeltaChunkVector* vec) +{ + if (vec->size == 0) + return; + + DeltaChunk* dc = vec->mem; + DeltaChunk* dcend = DCV_end(vec); + for(;dc < dcend; dc++){ + DC_destroy(dc); + } + + vec->size = 0; +} + // Append num-chunks to the end of the list, possibly reallocating existing ones // Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory @@ -316,7 +350,7 @@ static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) { if (vec->size + num_chunks > vec->reserved_size){ - DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size); + DCV_grow_by(vec, (vec->size + num_chunks) - vec->reserved_size); } Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; @@ -337,7 +371,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow(vec, 1); + DCV_grow_by(vec, 1); } DeltaChunk* next = vec->mem + vec->size; @@ -546,12 +580,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkVector bdcv; - DeltaChunkVector tdcv; DeltaChunkVector dcv; + DeltaChunkVector tdcv; DeltaChunkVector tmpl; - DCV_init(&bdcv, 0); - DCV_init(&dcv, 0); + DCV_init(&dcv, 100); // should be enough to keep the average text file DCV_init(&tdcv, 0); DCV_init(&tmpl, 200); @@ -578,7 +610,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // estimate number of ops - assume one third adds, half two byte (size+offset) copies const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); - DCV_grow(&dcv, approx_num_cmds); + DCV_reserve_memory(&dcv, approx_num_cmds); // parse command stream ull tbw = 0; // Amount of target bytes written @@ -632,16 +664,15 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!is_first_run){ DCV_connect_with_base(&tdcv, &dcv, &tmpl); } - // swap the vector - // Skip the first vector, as it is also used as top chunk vector - if (bdcv.mem != tdcv.mem){ - DCV_destroy(&bdcv); - } - bdcv = dcv; + if (is_first_run){ tdcv = dcv; + // wipe out dcv without destroying the members, get its own memory + DCV_init(&dcv, tdcv.size); + } else { + // destroy members, but keep memory + DCV_reset(&dcv); } - DCV_init(&dcv, 0); loop_end: // perform cleanup @@ -662,7 +693,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } DCV_destroy(&tmpl); - DCV_destroy(&bdcv); if (dsi > 1){ // otherwise dcv equals tcl DCV_destroy(&dcv); diff --git a/fun.py b/fun.py index e6262b4bc..13a3c627f 100644 --- a/fun.py +++ b/fun.py @@ -545,7 +545,6 @@ def connect_deltas(dstreams): :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" - bdcl = None # data chunk list for initial base tdcl = None # topmost dcl dcl = tdcl = TopdownDeltaChunkList() @@ -611,13 +610,12 @@ def connect_deltas(dstreams): dcl.compress() # merge the lists ! - if bdcl is not None: + if dsi > 0: if not tdcl.connect_with_next_base(dcl): break # END handle merge # prepare next base - bdcl = dcl dcl = DeltaChunkList() # END for each delta stream From 60f7768ed00ad666317c1877abe52906d6014d50 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 19:46:24 +0200 Subject: [PATCH 0085/3719] Implemented everything about the merging of the bases into the topmost delta list. Its not yet working, but at least its not crashing --- _fun.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/_fun.c b/_fun.c index 142795e6c..eb4b6b865 100644 --- a/_fun.c +++ b/_fun.c @@ -91,6 +91,9 @@ typedef unsigned int uint; typedef unsigned char uchar; typedef uchar bool; +// Constants +const ull gDVC_grow_by = 50; + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -277,10 +280,17 @@ DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) return &vec->mem[i]; } +// Return last item +inline +DeltaChunk* DCV_last(DeltaChunkVector* vec) +{ + return DCV_get(vec, vec->size-1); +} + inline ull DCV_rbound(DeltaChunkVector* vec) { - return DC_rbound(DCV_get(vec, vec->size-1)); + return DC_rbound(DCV_last(vec)); } inline @@ -371,7 +381,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, 1); + DCV_grow_by(vec, gDVC_grow_by); } DeltaChunk* next = vec->mem + vec->size; @@ -379,6 +389,33 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) return next; } +// Return delta chunk being closest to the given absolute offset +inline +DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) +{ + assert(vec->mem); + + ull lo = 0; + ull hi = vec->size; + ull mid; + DeltaChunk* dc; + + while (lo < hi) + { + mid = (lo + hi) / 2; + dc = vec->mem + mid; + if (dc->to > ofs){ + hi = mid; + } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + return dc; + } else { + lo = mid + 1; + } + } + + return DCV_last(vec); +} + // Write a slice as defined by its absolute offset in bytes and its size into the given // destination. The individual chunks written will be a deep copy of the source // data chunks @@ -387,14 +424,67 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) inline void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { + assert(DCV_lbound(src) <= ofs); + assert(DCV_rbound(src) <= ofs + size); + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + DeltaChunk* destc = DCV_append(dest); + const ull relofs = ofs - cdc->to; + DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + cdc += 1; + size -= destc->ts; + + if (size == 0){ + return; + } + } + + DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc) + { + if (cdc->ts < size) { + DC_copy_to(cdc, DCV_append(dest)); + size -= cdc->ts; + } else { + DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + size = 0; + break; + } + } + + assert(size == 0); } +// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which +// originates in to +// 'at' will be replaced by the items to insert ( special purpose ) +// 'at' will be properly destroyed, but all items will just be copied bytewise +// using memcpy. Hence from must just forget about them ! +inline +void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +{ + assert(from->size > 1); + + DCV_reserve_memory(to, to->size + from->size - 1); // -1 because we replace at + DC_destroy(at); + to->size -= 1 + from->size; + + // If we are somewhere in the middle, we have to make some space + if (DCV_last(to) != at) { + memmove((void*)at+from->size, (void*)(at+1), (size_t)(DCV_end(to) - (at+1))); + } + + // Finally copy all the items in + memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); +} + // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the // caller -static void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) { DeltaChunk* dc = tdcv->mem; @@ -413,18 +503,20 @@ void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, Delta // assert(tmpl->size); // move target bounds - DeltaChunk* cdc = tmpl->mem; - DeltaChunk* cdcend = tmpl->mem + tmpl->size; + DeltaChunk* tdc = tmpl->mem; + DeltaChunk* tdcend = tmpl->mem + tmpl->size; const ull ofs = dc->to - dc->so; - for(;cdc < cdcend; cdc++){ - cdc->to += ofs; + for(;tdc < tdcend; tdc++){ + tdc->to += ofs; } - // insert slice into our list, replacing our current chunk + // insert slice into our list if (tmpl->size == 1){ + // Its not data, so destroy is not really required, anyhow ... + DC_destroy(dc); *dc = *DCV_get(tmpl, 0); } else { - + DCV_replace_one_by_many(tmpl, tdcv, dc); } // make sure the members will not be deallocated by the list From 60b4f37a8b17c8f89b1b0ba7a0268f375469033d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 22:06:27 +0200 Subject: [PATCH 0086/3719] Improved performance of python implementation by 10 percent, just by removing function calls and object creations --- fun.py | 82 ++++++++++++---------------------------------------------- 1 file changed, 17 insertions(+), 65 deletions(-) diff --git a/fun.py b/fun.py index 13a3c627f..038b4c761 100644 --- a/fun.py +++ b/fun.py @@ -55,9 +55,6 @@ def _set_delta_rbound(d, size): :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size :return: d""" - if d.ts == size: - return - d.ts = size # NOTE: data is truncated automatically when applying the delta @@ -154,76 +151,30 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 -def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): +def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed write function. :param bbuf: base buffer containing the base of all deltas contained in this list. It will only be used if the chunk in question does not have a base chain. - :param lbound_offset: offset at which to start applying the delta, relative to - our lbound - :param size: if larger than 0, only the given amount of bytes will be applied :param write: function taking a string of bytes to write to the output""" - slen = len(dcl) - if slen == 0: - return - # END early abort - absofs = dcl.lbound() + lbound_offset - if size == 0: - size = dcl.rbound() - absofs - # END initialize size - - if lbound_offset or absofs + size != dcl.rbound(): - cdi = _closest_index(dcl, absofs) - cd = dcl[cdi] - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - delta_chunk_apply(tcd, bbuf, write) - size -= tcd.ts - cdi += 1 - # END handle first chunk - - # here we have to either apply full chunks, or smaller ones, but - # we always start at the chunks target offset - while cdi < slen and size: - cd = dcl[cdi] - if cd.ts <= size: - delta_chunk_apply(cd, bbuf, write) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - delta_chunk_apply(tcd, bbuf, write) - size -= tcd.ts - break - # END handle bytes to apply - cdi += 1 - # END handle rest - else: - for dc in dcl: - delta_chunk_apply(dc, bbuf, write) - # END for each dc - # END handle application values + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc -def delta_list_slice(dcl, absofs, size): +def delta_list_slice(dcl, absofs, size, ndcl): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. - :return: list (copy) which represents the given chunk""" - dcllbound = dcl.lbound() - absofs = max(absofs, dcllbound) - size = min(dcl.rbound() - dcllbound, size) + :return: None""" cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = list() lappend = ndcl.append if cd.to != absofs: - tcd = delta_duplicate(cd) + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) + tcd.ts = min(tcd.ts, size) lappend(tcd) size -= tcd.ts cdi += 1 @@ -233,11 +184,11 @@ def delta_list_slice(dcl, absofs, size): # are we larger than the current block cd = dcl[cdi] if cd.ts <= size: - lappend(delta_duplicate(cd)) + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) size -= cd.ts else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size lappend(tcd) size -= tcd.ts break @@ -245,7 +196,6 @@ def delta_list_slice(dcl, absofs, size): cdi += 1 # END for each chunk - return ndcl class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. @@ -272,10 +222,10 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def apply(self, bbuf, write, lbound_offset=0, size=0): + def apply(self, bbuf, write): """Only used by public clients, internally we only use the global routines for performance""" - return delta_list_apply(self, bbuf, write, lbound_offset, size) + return delta_list_apply(self, bbuf, write) def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate @@ -369,6 +319,7 @@ def connect_with_next_base(self, bdcl): nfc = 0 # number of frozen chunks dci = 0 # delta chunk index slen = len(self) # len of self + ccl = list() # temporary list while dci < slen: dc = self[dci] dci += 1 @@ -384,8 +335,9 @@ def connect_with_next_base(self, bdcl): # dont support efficient insertion ( just one at a time ), but for now # we live with it. Internally, its all just a 32/64bit pointer, and # the portions of moved memory should be smallish. Maybe we just rebuild - # ourselves in order to reduce the amount of insertions ... - ccl = delta_list_slice(bdcl, dc.so, dc.ts) + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) # move the target bounds into place to match with our chunk ofs = dc.to - dc.so From a63ee1d034a6677bc3ab408d1a16593bbb6078ca Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 23:55:45 +0200 Subject: [PATCH 0087/3719] Currently there is a weird memory bug, valgrind says it is writing one byte too much. Perhaps its because of the use of PyMem --- _fun.c | 115 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/_fun.c b/_fun.c index eb4b6b865..fc4ee1edc 100644 --- a/_fun.c +++ b/_fun.c @@ -94,6 +94,12 @@ typedef uchar bool; // Constants const ull gDVC_grow_by = 50; +#ifdef DEBUG +#define DBG_check(vec) DCV_dbg_check_integrity(vec) +#else +#define DBG_check(vec) +#endif + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -154,19 +160,20 @@ void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared } inline -ull DC_rbound(DeltaChunk* dc) +ull DC_rbound(const DeltaChunk* dc) { return dc->to + dc->ts; } // Copy all data from src to dest, the data pointer will be copied too inline -void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) +void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) { dest->to = src->to; dest->ts = src->ts; dest->so = src->so; dest->data_shared = 0; + dest->data = NULL; DC_set_data(dest, src->data, src->ts, 0); } @@ -174,7 +181,7 @@ void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) // Copy all data with the given offset and size. The source offset, as well // as the data will be truncated accordingly inline -void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) { assert(size <= src->ts); assert(src->to + ofs + size <= DC_rbound(src)); @@ -182,6 +189,7 @@ void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) dest->to = src->to + ofs; dest->ts = size; dest->so = src->so + ofs; + dest->data = NULL; if (src->data){ DC_set_data(dest, src->data + ofs, size, 0); @@ -260,13 +268,13 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) } inline -ull DCV_len(DeltaChunkVector* vec) +ull DCV_len(const DeltaChunkVector* vec) { return vec->size; } inline -ull DCV_lbound(DeltaChunkVector* vec) +ull DCV_lbound(const DeltaChunkVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -274,7 +282,7 @@ ull DCV_lbound(DeltaChunkVector* vec) // Return item at index inline -DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) +DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; @@ -282,29 +290,29 @@ DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) // Return last item inline -DeltaChunk* DCV_last(DeltaChunkVector* vec) +DeltaChunk* DCV_last(const DeltaChunkVector* vec) { return DCV_get(vec, vec->size-1); } inline -ull DCV_rbound(DeltaChunkVector* vec) +ull DCV_rbound(const DeltaChunkVector* vec) { return DC_rbound(DCV_last(vec)); } inline -int DCV_empty(DeltaChunkVector* vec) +int DCV_empty(const DeltaChunkVector* vec) { return vec->size == 0; } // Return end pointer of the vector inline -DeltaChunk* DCV_end(DeltaChunkVector* vec) +const DeltaChunk* DCV_end(const DeltaChunkVector* vec) { assert(!DCV_empty(vec)); - return &vec->mem[vec->size]; + return vec->mem + vec->size; } void DCV_destroy(DeltaChunkVector* vec) @@ -345,7 +353,7 @@ void DCV_reset(DeltaChunkVector* vec) return; DeltaChunk* dc = vec->mem; - DeltaChunk* dcend = DCV_end(vec); + const DeltaChunk* dcend = DCV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); } @@ -366,11 +374,9 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_ssize_t old_size = vec->size; vec->size += num_chunks; -#ifdef DEBUG for(;old_size < vec->size; ++old_size){ DC_init(DCV_get(vec, old_size), 0, 0, 0); } -#endif return &vec->mem[old_size]; } @@ -391,7 +397,7 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) // Return delta chunk being closest to the given absolute offset inline -DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) +DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) { assert(vec->mem); @@ -416,16 +422,43 @@ DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) return DCV_last(vec); } +// Assert the given vector has correct datachunks +void DCV_dbg_check_integrity(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + const DeltaChunk* i = vec->mem; + const DeltaChunk* end = DCV_end(vec); + + ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); + ull acc_size = 0; + for(; i < end; i++){ + acc_size += i->ts; + } + assert(acc_size == aparent_size); + + if (vec->size < 2){ + return; + } + + const DeltaChunk* endm1 = DCV_end(vec) - 1; + for(i = vec->mem; i < endm1; i++){ + const DeltaChunk* n = i+1; + assert(DC_rbound(i) == n->to); + } + +} + // Write a slice as defined by its absolute offset in bytes and its size into the given // destination. The individual chunks written will be a deep copy of the source // data chunks // TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort // of append-only memory pool would improve performance inline -void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { + //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); assert(DCV_lbound(src) <= ofs); - assert(DCV_rbound(src) <= ofs + size); + assert((ofs + size) <= DCV_rbound(src)); DeltaChunk* cdc = DCV_closest_chunk(src, ofs); @@ -442,7 +475,7 @@ void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, u } } - DeltaChunk* vecend = DCV_end(src); + const DeltaChunk* vecend = DCV_end(src); for( ;(cdc < vecend) && size; ++cdc) { if (cdc->ts < size) { @@ -464,18 +497,22 @@ void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, u // 'at' will be replaced by the items to insert ( special purpose ) // 'at' will be properly destroyed, but all items will just be copied bytewise // using memcpy. Hence from must just forget about them ! +// IMPORTANT: to must have an appropriate size already inline -void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { + fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); + assert(to->size + from->size - 1 <= to->reserved_size); - DCV_reserve_memory(to, to->size + from->size - 1); // -1 because we replace at + // -1 because we replace 'at' DC_destroy(at); - to->size -= 1 + from->size; + to->size += from->size - 1; // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - memmove((void*)at+from->size, (void*)(at+1), (size_t)(DCV_end(to) - (at+1))); + fprintf(stderr, "moving to %p from %p, num bytes = %i\n", at+from->size, at+1, (int)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)(DCV_end(to) - (at+1)) * sizeof(DeltaChunk)); } // Finally copy all the items in @@ -485,22 +522,27 @@ void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, Delta // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the // caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) { - DeltaChunk* dc = tdcv->mem; - DeltaChunk* end = tdcv->mem + tdcv->size; - assert(dc); + Py_ssize_t dci = 0; + Py_ssize_t iend = tdcv->size; + DeltaChunk* dc; - for (;dc < end; dc++) + DBG_check(tdcv); + DBG_check(bdcv); + + for (;dci < iend; dci++) { // Data chunks don't need processing + dc = DCV_get(tdcv, dci); if (dc->data){ continue; } // Copy Chunk Handling DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - // assert(tmpl->size); + DBG_check(tmpl); + assert(tmpl->size); // move target bounds DeltaChunk* tdc = tmpl->mem; @@ -516,8 +558,15 @@ void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, Delta DC_destroy(dc); *dc = *DCV_get(tmpl, 0); } else { + DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); + dc = DCV_get(tdcv, dci); DCV_replace_one_by_many(tmpl, tdcv, dc); + // Compensate for us being replaced + dci += tmpl->size-1; + iend += tmpl->size-1; } + + DBG_check(tdcv); // make sure the members will not be deallocated by the list DCV_forget_members(tmpl); @@ -679,8 +728,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DCV_init(&tdcv, 0); DCV_init(&tmpl, 200); - unsigned int dsi; - PyObject* ds; + unsigned int dsi = 0; + PyObject* ds = 0; int error = 0; for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) { @@ -706,7 +755,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // parse command stream ull tbw = 0; // Amount of target bytes written - bool shared_data = dsi != 0; + bool is_shared_data = dsi != 0; bool is_first_run = dsi == 0; assert(data < dend); @@ -742,10 +791,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // What's faster DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd, shared_data); + DC_set_data(dc, data, cmd, is_shared_data); tbw += cmd; data += cmd; - } else { + } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); goto loop_end; From 4098056f4fe101f1e50d924ab3ba40650d563b9d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 00:14:06 +0200 Subject: [PATCH 0088/3719] Fixed terrible bug, which happened due to a change of the size of the vector, but too early actually, so a memmove would use incorrect values --- _fun.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/_fun.c b/_fun.c index fc4ee1edc..3be3bccab 100644 --- a/_fun.c +++ b/_fun.c @@ -501,22 +501,24 @@ void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull inline void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { - fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); + //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); assert(to->size + from->size - 1 <= to->reserved_size); // -1 because we replace 'at' DC_destroy(at); - to->size += from->size - 1; // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - fprintf(stderr, "moving to %p from %p, num bytes = %i\n", at+from->size, at+1, (int)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - memmove((void*)(at+from->size), (void*)(at+1), (size_t)(DCV_end(to) - (at+1)) * sizeof(DeltaChunk)); + //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } - + // Finally copy all the items in memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + + // FINALLY: update size + to->size += from->size - 1; } // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost From f563a9db73a24c6353794c8e093cc604be9c7d7b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 00:49:27 +0200 Subject: [PATCH 0089/3719] Fixed integrity check function, finalized code, so far it is working, and its very efficient as well as the amount of data-copies is minimized --- _fun.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/_fun.c b/_fun.c index 3be3bccab..0b485473f 100644 --- a/_fun.c +++ b/_fun.c @@ -95,7 +95,7 @@ typedef uchar bool; const ull gDVC_grow_by = 50; #ifdef DEBUG -#define DBG_check(vec) DCV_dbg_check_integrity(vec) +#define DBG_check(vec) asser(DCV_dbg_check_integrity(vec)) #else #define DBG_check(vec) #endif @@ -165,6 +165,26 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } +// Apply +inline +void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) +{ + PyObject* buffer = 0; + if (dc->data){ + buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); + } else { + buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); + } + + if (PyTuple_SetItem(tmpargs, 0, buffer)){ + assert(0); + } + + // tuple steals reference, and will take care about the deallocation + PyObject_Call(writer, tmpargs, NULL); + +} + // Copy all data from src to dest, the data pointer will be copied too inline void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) @@ -423,9 +443,12 @@ DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) } // Assert the given vector has correct datachunks -void DCV_dbg_check_integrity(const DeltaChunkVector* vec) +// return 1 on success +int DCV_dbg_check_integrity(const DeltaChunkVector* vec) { - assert(!DCV_empty(vec)); + if(DCV_empty(vec)){ + return 0; + } const DeltaChunk* i = vec->mem; const DeltaChunk* end = DCV_end(vec); @@ -434,18 +457,22 @@ void DCV_dbg_check_integrity(const DeltaChunkVector* vec) for(; i < end; i++){ acc_size += i->ts; } - assert(acc_size == aparent_size); + if (acc_size != aparent_size) + return 0; if (vec->size < 2){ - return; + return 1; } const DeltaChunk* endm1 = DCV_end(vec) - 1; for(i = vec->mem; i < endm1; i++){ const DeltaChunk* n = i+1; - assert(DC_rbound(i) == n->to); + if (DC_rbound(i) != n->to){ + return 0; + } } + return 1; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -624,10 +651,41 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) return PyLong_FromUnsignedLongLong(DCL_rbound(self)); } +// Write using a write function, taking remaining bytes from a base buffer static -PyObject* DCL_apply(PyObject* self, PyObject* args) +PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + PyObject* pybuf = 0; + PyObject* writeproc = 0; + if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ + PyErr_BadArgument(); + return NULL; + } + + if (!PyObject_CheckReadBuffer(pybuf)){ + PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + + if (!PyCallable_Check(writeproc)){ + PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); + return NULL; + } + + const DeltaChunk* i = self->vec.mem; + const DeltaChunk* end = DCV_end(&self->vec); + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + + PyObject* tmpargs = PyTuple_New(1); + + for(; i < end; i++){ + DC_apply(i, data, writeproc, tmpargs); + } + Py_DECREF(tmpargs); Py_RETURN_NONE; } From 64992444e9a2b2936cea4d2cba235e59e703cac9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 10:46:21 +0200 Subject: [PATCH 0090/3719] optimized reallocation count, which improves speed a little bit. Previously it would easily get into the habbit of reallocating the vector just to add a single item --- _fun.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/_fun.c b/_fun.c index 0b485473f..1373a5c60 100644 --- a/_fun.c +++ b/_fun.c @@ -92,10 +92,10 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDVC_grow_by = 50; +const ull gDVC_grow_by = 100; #ifdef DEBUG -#define DBG_check(vec) asser(DCV_dbg_check_integrity(vec)) +#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) #else #define DBG_check(vec) #endif @@ -233,6 +233,8 @@ typedef struct { // Reserve enough memory to hold the given amount of delta chunks // Return 1 on success +// NOTE: added a minimum allocation to assure reallocation is not done +// just for a single additional entry. DCVs change often, and reallocs are expensive inline int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) { @@ -240,6 +242,10 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) return 1; } + if (num_dc - vec->reserved_size){ + num_dc += gDVC_grow_by; + } + #ifdef DEBUG bool was_null = vec->mem == NULL; #endif @@ -381,25 +387,6 @@ void DCV_reset(DeltaChunkVector* vec) vec->size = 0; } -// Append num-chunks to the end of the list, possibly reallocating existing ones -// Return a pointer to the first of the added items. They are already null initialized -// If num-chunks == 0, it returns the end pointer of the allocated memory -static inline -DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) -{ - if (vec->size + num_chunks > vec->reserved_size){ - DCV_grow_by(vec, (vec->size + num_chunks) - vec->reserved_size); - } - Py_FatalError("Could not allocate memory for append operation"); - Py_ssize_t old_size = vec->size; - vec->size += num_chunks; - - for(;old_size < vec->size; ++old_size){ - DC_init(DCV_get(vec, old_size), 0, 0, 0); - } - - return &vec->mem[old_size]; -} // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! @@ -838,6 +825,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ + assert(0); break; } @@ -849,6 +837,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // NOTE: Compression only necessary for all other deltas, not // for the first one, as we will share the data. It really depends // What's faster + // Compression reduces fragmentation though, which is why we do it + // in all cases. DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); DC_set_data(dc, data, cmd, is_shared_data); @@ -860,7 +850,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } }// END handle command opcodes - assert(tbw == target_size); + if (tbw != target_size){ + PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); + error = 1; + } if (!is_first_run){ DCV_connect_with_base(&tdcv, &dcv, &tmpl); From a7253b8d08ffc69bf3afdec98619044c84c64f75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 12:19:06 +0200 Subject: [PATCH 0091/3719] implemented memory compression, but got evil memory bug once again ... probably its just as stupid as previously --- _fun.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/_fun.c b/_fun.c index 1373a5c60..cf1f5b629 100644 --- a/_fun.c +++ b/_fun.c @@ -159,6 +159,16 @@ void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared } +// Make the given data our own. It is assumed to have the size stored in our instance +// and will be managed by us. +inline +void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) +{ + assert(data); + DC_deallocate_data(dc); + dc->data = data; +} + inline ull DC_rbound(const DeltaChunk* dc) { @@ -214,7 +224,6 @@ void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull siz if (src->data){ DC_set_data(dest, src->data + ofs, size, 0); } else { - dest->data = NULL; dest->data_shared = 0; } } @@ -825,6 +834,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ + // this really shouldn't happen + error = 1; assert(0); break; } @@ -834,16 +845,53 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } else if (cmd) { // TODO: Compress nodes by parsing them in advance - // NOTE: Compression only necessary for all other deltas, not - // for the first one, as we will share the data. It really depends - // What's faster // Compression reduces fragmentation though, which is why we do it // in all cases. - DeltaChunk* dc = DCV_append(&dcv); - DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd, is_shared_data); - tbw += cmd; + const uchar* add_start = data - 1; + const uchar* add_end = dend; + ull num_bytes = cmd; data += cmd; + ull num_chunks = 1; + while (data < dend){ + fprintf(stderr, "looping\n"); + const char c = *data; + if (c & 0x80){ + add_end = data; + break; + } else { + num_chunks += 1; + data += c + 1; // advance by 1 to skip add cmd + num_bytes += c; + } + } + + fprintf(stderr, "add bytes = %i\n", (int)num_bytes); + #ifdef DEBUG + assert(add_end - add_start > 0); + if (num_chunks > 1){ + fprintf(stderr, "Compression worked, got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + } + #endif + + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, num_bytes, 0); + + // gather the data, or (possibly) share single blocks + if (num_chunks > 1){ + uchar* dcdata = PyMem_Malloc(num_bytes); + while (add_start < add_end){ + const char bytes = *add_start++; + fprintf(stderr, "Copying %i bytes\n", bytes); + memcpy((void*)dcdata, (void*)add_start, bytes); + dcdata += bytes; + add_start += bytes; + } + DC_set_data_with_ownership(dc, dcdata); + } else { + DC_set_data(dc, data - cmd, cmd, is_shared_data); + } + + tbw += num_bytes; } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); From bd01f6932c12fe2b7010884d339cc4831cc7ec59 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:27:38 +0200 Subject: [PATCH 0092/3719] Fixed memory bug, it was a small tiny thing, as well as stupid. --- _fun.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/_fun.c b/_fun.c index cf1f5b629..247008682 100644 --- a/_fun.c +++ b/_fun.c @@ -844,32 +844,32 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) tbw += cp_size; } else if (cmd) { - // TODO: Compress nodes by parsing them in advance // Compression reduces fragmentation though, which is why we do it // in all cases. + // It makes the more sense the more consecutive add-chunks we have, + // its more likely in big deltas, for big binary files const uchar* add_start = data - 1; const uchar* add_end = dend; ull num_bytes = cmd; data += cmd; ull num_chunks = 1; while (data < dend){ - fprintf(stderr, "looping\n"); + //while (0){ const char c = *data; if (c & 0x80){ add_end = data; break; } else { - num_chunks += 1; - data += c + 1; // advance by 1 to skip add cmd + data += 1 + c; // advance by 1 to skip add cmd num_bytes += c; + num_chunks += 1; } } - fprintf(stderr, "add bytes = %i\n", (int)num_bytes); #ifdef DEBUG assert(add_end - add_start > 0); if (num_chunks > 1){ - fprintf(stderr, "Compression worked, got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); } #endif @@ -881,12 +881,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) uchar* dcdata = PyMem_Malloc(num_bytes); while (add_start < add_end){ const char bytes = *add_start++; - fprintf(stderr, "Copying %i bytes\n", bytes); memcpy((void*)dcdata, (void*)add_start, bytes); dcdata += bytes; add_start += bytes; } - DC_set_data_with_ownership(dc, dcdata); + DC_set_data_with_ownership(dc, dcdata-num_bytes); } else { DC_set_data(dc, data - cmd, cmd, is_shared_data); } From 5442e4aec5741d5b346ab9a7cd091976a8f5e9f4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:48:44 +0200 Subject: [PATCH 0093/3719] Put delta-apply code into separate function. Would have preferred to to have just one dynamic module, lets see whether includes are possible --- Makefile | 3 + _delta_apply.c | 903 +++++++++++++++++++++++++++++++++++++++++++++++++ _fun.c | 885 ------------------------------------------------ fun.py | 2 +- setup.py | 5 +- 5 files changed, 911 insertions(+), 887 deletions(-) create mode 100644 _delta_apply.c diff --git a/Makefile b/Makefile index 190a66b03..e65c55a6d 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ build:: $(SETUP) build $(SETUP) build_ext -i +build_ext:: + $(SETUP) build_ext -i + install:: $(SETUP) install diff --git a/_delta_apply.c b/_delta_apply.c new file mode 100644 index 000000000..40ea60c56 --- /dev/null +++ b/_delta_apply.c @@ -0,0 +1,903 @@ +#include +#include +#include +#include +#include +#include + +typedef unsigned long long ull; +typedef unsigned int uint; +typedef unsigned char uchar; +typedef uchar bool; + +// Constants +const ull gDVC_grow_by = 100; + +#ifdef DEBUG +#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) +#else +#define DBG_check(vec) +#endif + +// DELTA CHUNK +//////////////// +// Internal Delta Chunk Objects +typedef struct { + ull to; + ull ts; + ull so; + const uchar* data; + bool data_shared; +} DeltaChunk; + +inline +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) +{ + dc->to = to; + dc->ts = ts; + dc->so = so; + dc->data = NULL; + dc->data_shared = 0; +} + +inline +void DC_deallocate_data(DeltaChunk* dc) +{ + if (!dc->data_shared && dc->data){ + PyMem_Free((void*)dc->data); + } + dc->data = NULL; +} + +inline +void DC_destroy(DeltaChunk* dc) +{ + DC_deallocate_data(dc); +} + +// Store a copy of data in our instance. If shared is 1, the data will be shared, +// hence it will only be stored, but the memory will not be touched, or copied. +inline +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) +{ + DC_deallocate_data(dc); + + if (data == 0){ + dc->data = NULL; + dc->data_shared = 0; + return; + } + + dc->data_shared = shared; + if (shared){ + dc->data = data; + } else { + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy((void*)dc->data, (void*)data, dlen); + } + +} + +// Make the given data our own. It is assumed to have the size stored in our instance +// and will be managed by us. +inline +void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) +{ + assert(data); + DC_deallocate_data(dc); + dc->data = data; +} + +inline +ull DC_rbound(const DeltaChunk* dc) +{ + return dc->to + dc->ts; +} + +// Apply +inline +void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) +{ + PyObject* buffer = 0; + if (dc->data){ + buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); + } else { + buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); + } + + if (PyTuple_SetItem(tmpargs, 0, buffer)){ + assert(0); + } + + // tuple steals reference, and will take care about the deallocation + PyObject_Call(writer, tmpargs, NULL); + +} + +// Copy all data from src to dest, the data pointer will be copied too +inline +void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) +{ + dest->to = src->to; + dest->ts = src->ts; + dest->so = src->so; + dest->data_shared = 0; + dest->data = NULL; + + DC_set_data(dest, src->data, src->ts, 0); +} + +// Copy all data with the given offset and size. The source offset, as well +// as the data will be truncated accordingly +inline +void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +{ + assert(size <= src->ts); + assert(src->to + ofs + size <= DC_rbound(src)); + + dest->to = src->to + ofs; + dest->ts = size; + dest->so = src->so + ofs; + dest->data = NULL; + + if (src->data){ + DC_set_data(dest, src->data + ofs, size, 0); + } else { + dest->data_shared = 0; + } +} + + +// DELTA CHUNK VECTOR +///////////////////// + +typedef struct { + DeltaChunk* mem; // Memory + Py_ssize_t size; // Size in DeltaChunks + Py_ssize_t reserved_size; // Reserve in DeltaChunks +} DeltaChunkVector; + + + +// Reserve enough memory to hold the given amount of delta chunks +// Return 1 on success +// NOTE: added a minimum allocation to assure reallocation is not done +// just for a single additional entry. DCVs change often, and reallocs are expensive +inline +int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) +{ + if (num_dc <= vec->reserved_size){ + return 1; + } + + if (num_dc - vec->reserved_size){ + num_dc += gDVC_grow_by; + } + +#ifdef DEBUG + bool was_null = vec->mem == NULL; +#endif + + if (vec->mem == NULL){ + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); + } else { + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); + } + + if (vec->mem == NULL){ + Py_FatalError("Could not allocate memory for append operation"); + } + + vec->reserved_size = num_dc; + +#ifdef DEBUG + const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; + if (!was_null) + format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); +#endif + + return vec->mem != NULL; +} + +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +inline +int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +{ + return DCV_reserve_memory(vec, vec->reserved_size + num_dc); +} + +int DCV_init(DeltaChunkVector* vec, ull initial_size) +{ + vec->mem = NULL; + vec->size = 0; + vec->reserved_size = 0; + + return DCV_grow_by(vec, initial_size); +} + +inline +ull DCV_len(const DeltaChunkVector* vec) +{ + return vec->size; +} + +inline +ull DCV_lbound(const DeltaChunkVector* vec) +{ + assert(vec->size && vec->mem); + return vec->mem->to; +} + +// Return item at index +inline +DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) +{ + assert(i < vec->size && vec->mem); + return &vec->mem[i]; +} + +// Return last item +inline +DeltaChunk* DCV_last(const DeltaChunkVector* vec) +{ + return DCV_get(vec, vec->size-1); +} + +inline +ull DCV_rbound(const DeltaChunkVector* vec) +{ + return DC_rbound(DCV_last(vec)); +} + +inline +int DCV_empty(const DeltaChunkVector* vec) +{ + return vec->size == 0; +} + +// Return end pointer of the vector +inline +const DeltaChunk* DCV_end(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return vec->mem + vec->size; +} + +void DCV_destroy(DeltaChunkVector* vec) +{ + if (vec->mem){ +#ifdef DEBUG + fprintf(stderr, "Freeing %p\n", (void*)vec->mem); +#endif + + const DeltaChunk* end = &vec->mem[vec->size]; + DeltaChunk* i; + for(i = vec->mem; i < end; i++){ + DC_destroy(i); + } + + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +// Reset this vector so that its existing memory can be filled again. +// Memory will be kept, but not cleaned up +inline +void DCV_forget_members(DeltaChunkVector* vec) +{ + vec->size = 0; +} + +// Reset the vector so that its size will be zero, and its members will +// have been deallocated properly. +// It will keep its memory though, and hence can be filled again +inline +void DCV_reset(DeltaChunkVector* vec) +{ + if (vec->size == 0) + return; + + DeltaChunk* dc = vec->mem; + const DeltaChunk* dcend = DCV_end(vec); + for(;dc < dcend; dc++){ + DC_destroy(dc); + } + + vec->size = 0; +} + + +// Append one chunk to the end of the list, and return a pointer to it +// It will not have been initialized ! +static inline +DeltaChunk* DCV_append(DeltaChunkVector* vec) +{ + if (vec->size + 1 > vec->reserved_size){ + DCV_grow_by(vec, gDVC_grow_by); + } + + DeltaChunk* next = vec->mem + vec->size; + vec->size += 1; + return next; +} + +// Return delta chunk being closest to the given absolute offset +inline +DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) +{ + assert(vec->mem); + + ull lo = 0; + ull hi = vec->size; + ull mid; + DeltaChunk* dc; + + while (lo < hi) + { + mid = (lo + hi) / 2; + dc = vec->mem + mid; + if (dc->to > ofs){ + hi = mid; + } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + return dc; + } else { + lo = mid + 1; + } + } + + return DCV_last(vec); +} + +// Assert the given vector has correct datachunks +// return 1 on success +int DCV_dbg_check_integrity(const DeltaChunkVector* vec) +{ + if(DCV_empty(vec)){ + return 0; + } + const DeltaChunk* i = vec->mem; + const DeltaChunk* end = DCV_end(vec); + + ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); + ull acc_size = 0; + for(; i < end; i++){ + acc_size += i->ts; + } + if (acc_size != aparent_size) + return 0; + + if (vec->size < 2){ + return 1; + } + + const DeltaChunk* endm1 = DCV_end(vec) - 1; + for(i = vec->mem; i < endm1; i++){ + const DeltaChunk* n = i+1; + if (DC_rbound(i) != n->to){ + return 0; + } + } + + return 1; +} + +// Write a slice as defined by its absolute offset in bytes and its size into the given +// destination. The individual chunks written will be a deep copy of the source +// data chunks +// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort +// of append-only memory pool would improve performance +inline +void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +{ + //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); + assert(DCV_lbound(src) <= ofs); + assert((ofs + size) <= DCV_rbound(src)); + + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + DeltaChunk* destc = DCV_append(dest); + const ull relofs = ofs - cdc->to; + DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + cdc += 1; + size -= destc->ts; + + if (size == 0){ + return; + } + } + + const DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc) + { + if (cdc->ts < size) { + DC_copy_to(cdc, DCV_append(dest)); + size -= cdc->ts; + } else { + DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + size = 0; + break; + } + } + + assert(size == 0); +} + + +// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which +// originates in to +// 'at' will be replaced by the items to insert ( special purpose ) +// 'at' will be properly destroyed, but all items will just be copied bytewise +// using memcpy. Hence from must just forget about them ! +// IMPORTANT: to must have an appropriate size already +inline +void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +{ + //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); + assert(from->size > 1); + assert(to->size + from->size - 1 <= to->reserved_size); + + // -1 because we replace 'at' + DC_destroy(at); + + // If we are somewhere in the middle, we have to make some space + if (DCV_last(to) != at) { + //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); + } + + // Finally copy all the items in + memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + + // FINALLY: update size + to->size += from->size - 1; +} + +// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost +// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the +// caller +void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +{ + Py_ssize_t dci = 0; + Py_ssize_t iend = tdcv->size; + DeltaChunk* dc; + + DBG_check(tdcv); + DBG_check(bdcv); + + for (;dci < iend; dci++) + { + // Data chunks don't need processing + dc = DCV_get(tdcv, dci); + if (dc->data){ + continue; + } + + // Copy Chunk Handling + DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); + DBG_check(tmpl); + assert(tmpl->size); + + // move target bounds + DeltaChunk* tdc = tmpl->mem; + DeltaChunk* tdcend = tmpl->mem + tmpl->size; + const ull ofs = dc->to - dc->so; + for(;tdc < tdcend; tdc++){ + tdc->to += ofs; + } + + // insert slice into our list + if (tmpl->size == 1){ + // Its not data, so destroy is not really required, anyhow ... + DC_destroy(dc); + *dc = *DCV_get(tmpl, 0); + } else { + DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); + dc = DCV_get(tdcv, dci); + DCV_replace_one_by_many(tmpl, tdcv, dc); + // Compensate for us being replaced + dci += tmpl->size-1; + iend += tmpl->size-1; + } + + DBG_check(tdcv); + + // make sure the members will not be deallocated by the list + DCV_forget_members(tmpl); + } +} + +// DELTA CHUNK LIST (PYTHON) +///////////////////////////// + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunkVector vec; + +} DeltaChunkList; + + +static +int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) +{ + if(args && PySequence_Size(args) > 0){ + PyErr_SetString(PyExc_ValueError, "Too many arguments"); + return -1; + } + + DCV_init(&self->vec, 0); + return 0; +} + +static +void DCL_dealloc(DeltaChunkList* self) +{ + DCV_destroy(&(self->vec)); +} + +static +PyObject* DCL_len(DeltaChunkList* self) +{ + return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); +} + +static inline +ull DCL_rbound(DeltaChunkList* self) +{ + if (DCV_empty(&self->vec)) + return 0; + return DCV_rbound(&self->vec); +} + +static +PyObject* DCL_py_rbound(DeltaChunkList* self) +{ + return PyLong_FromUnsignedLongLong(DCL_rbound(self)); +} + +// Write using a write function, taking remaining bytes from a base buffer +static +PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) +{ + PyObject* pybuf = 0; + PyObject* writeproc = 0; + if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ + PyErr_BadArgument(); + return NULL; + } + + if (!PyObject_CheckReadBuffer(pybuf)){ + PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + + if (!PyCallable_Check(writeproc)){ + PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); + return NULL; + } + + const DeltaChunk* i = self->vec.mem; + const DeltaChunk* end = DCV_end(&self->vec); + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + + PyObject* tmpargs = PyTuple_New(1); + + for(; i < end; i++){ + DC_apply(i, data, writeproc, tmpargs); + } + + Py_DECREF(tmpargs); + Py_RETURN_NONE; +} + +static PyMethodDef DCL_methods[] = { + {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, + {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject DeltaChunkListType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "DeltaChunkList", /*tp_name*/ + sizeof(DeltaChunkList), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)DCL_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "Minimal Delta Chunk List",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DCL_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)DCL_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +// Makes a new copy of the DeltaChunkList - you have to do everything yourselve +// in C ... want C++ !! +DeltaChunkList* DCL_new_instance(void) +{ + DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); + assert(dcl); + + DCL_init(dcl, 0, 0); + assert(dcl->vec.size == 0); + assert(dcl->vec.mem == NULL); + return dcl; +} + +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; + return size; +} + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +{ + // obtain iterator + PyObject* stream_iter = 0; + if (!PyIter_Check(dstreams)){ + stream_iter = PyObject_GetIter(dstreams); + if (!stream_iter){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); + return NULL; + } + } else { + stream_iter = dstreams; + } + + DeltaChunkVector dcv; + DeltaChunkVector tdcv; + DeltaChunkVector tmpl; + DCV_init(&dcv, 100); // should be enough to keep the average text file + DCV_init(&tdcv, 0); + DCV_init(&tmpl, 200); + + unsigned int dsi = 0; + PyObject* ds = 0; + int error = 0; + for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + { + PyObject* db = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(db)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); + goto loop_end; + } + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + const uchar* dend = data + dlen; + + // read header + const ull base_size = msb_size(&data, dend); + const ull target_size = msb_size(&data, dend); + + // estimate number of ops - assume one third adds, half two byte (size+offset) copies + const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + DCV_reserve_memory(&dcv, approx_num_cmds); + + // parse command stream + ull tbw = 0; // Amount of target bytes written + bool is_shared_data = dsi != 0; + bool is_first_run = dsi == 0; + + assert(data < dend); + while (data < dend) + { + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + const unsigned long rbound = cp_off + cp_size; + if (rbound < cp_size || + rbound > base_size){ + // this really shouldn't happen + error = 1; + assert(0); + break; + } + + DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); + tbw += cp_size; + + } else if (cmd) { + // Compression reduces fragmentation though, which is why we do it + // in all cases. + // It makes the more sense the more consecutive add-chunks we have, + // its more likely in big deltas, for big binary files + const uchar* add_start = data - 1; + const uchar* add_end = dend; + ull num_bytes = cmd; + data += cmd; + ull num_chunks = 1; + while (data < dend){ + //while (0){ + const char c = *data; + if (c & 0x80){ + add_end = data; + break; + } else { + data += 1 + c; // advance by 1 to skip add cmd + num_bytes += c; + num_chunks += 1; + } + } + + #ifdef DEBUG + assert(add_end - add_start > 0); + if (num_chunks > 1){ + fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + } + #endif + + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, num_bytes, 0); + + // gather the data, or (possibly) share single blocks + if (num_chunks > 1){ + uchar* dcdata = PyMem_Malloc(num_bytes); + while (add_start < add_end){ + const char bytes = *add_start++; + memcpy((void*)dcdata, (void*)add_start, bytes); + dcdata += bytes; + add_start += bytes; + } + DC_set_data_with_ownership(dc, dcdata-num_bytes); + } else { + DC_set_data(dc, data - cmd, cmd, is_shared_data); + } + + tbw += num_bytes; + } else { + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + goto loop_end; + } + }// END handle command opcodes + if (tbw != target_size){ + PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); + error = 1; + } + + if (!is_first_run){ + DCV_connect_with_base(&tdcv, &dcv, &tmpl); + } + + if (is_first_run){ + tdcv = dcv; + // wipe out dcv without destroying the members, get its own memory + DCV_init(&dcv, tdcv.size); + } else { + // destroy members, but keep memory + DCV_reset(&dcv); + } + +loop_end: + // perform cleanup + Py_DECREF(ds); + Py_DECREF(db); + + if (error){ + break; + } + }// END for each stream object + + if (dsi == 0 && ! error){ + PyErr_SetString(PyExc_ValueError, "No streams provided"); + } + + if (stream_iter != dstreams){ + Py_DECREF(stream_iter); + } + + DCV_destroy(&tmpl); + if (dsi > 1){ + // otherwise dcv equals tcl + DCV_destroy(&dcv); + } + + // Return the actual python object - its just a container + DeltaChunkList* dcl = DCL_new_instance(); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + // Otherwise tdcv would be deallocated by the chunk list + DCV_destroy(&tdcv); + error = 1; + } else { + // Plain copy, don't deallocate + dcl->vec = tdcv; + } + + if (error){ + // Will dealloc tdcv + Py_XDECREF(dcl); + return NULL; + } + + return (PyObject*)dcl; +} + +static PyMethodDef py_fun[] = { + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, + { NULL, NULL, 0, NULL } +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC init_delta_apply(void) +{ + PyObject *m; + + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + + m = Py_InitModule3("_delta_apply", py_fun, NULL); + if (m == NULL) + return; + + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); +} diff --git a/_fun.c b/_fun.c index 247008682..1881bfb05 100644 --- a/_fun.c +++ b/_fun.c @@ -1,9 +1,4 @@ #include -#include -#include -#include -#include -#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -86,883 +81,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) } -typedef unsigned long long ull; -typedef unsigned int uint; -typedef unsigned char uchar; -typedef uchar bool; - -// Constants -const ull gDVC_grow_by = 100; - -#ifdef DEBUG -#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) -#else -#define DBG_check(vec) -#endif - -// DELTA CHUNK -//////////////// -// Internal Delta Chunk Objects -typedef struct { - ull to; - ull ts; - ull so; - const uchar* data; - bool data_shared; -} DeltaChunk; - -inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) -{ - dc->to = to; - dc->ts = ts; - dc->so = so; - dc->data = NULL; - dc->data_shared = 0; -} - -inline -void DC_deallocate_data(DeltaChunk* dc) -{ - if (!dc->data_shared && dc->data){ - PyMem_Free((void*)dc->data); - } - dc->data = NULL; -} - -inline -void DC_destroy(DeltaChunk* dc) -{ - DC_deallocate_data(dc); -} - -// Store a copy of data in our instance. If shared is 1, the data will be shared, -// hence it will only be stored, but the memory will not be touched, or copied. -inline -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) -{ - DC_deallocate_data(dc); - - if (data == 0){ - dc->data = NULL; - dc->data_shared = 0; - return; - } - - dc->data_shared = shared; - if (shared){ - dc->data = data; - } else { - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy((void*)dc->data, (void*)data, dlen); - } - -} - -// Make the given data our own. It is assumed to have the size stored in our instance -// and will be managed by us. -inline -void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) -{ - assert(data); - DC_deallocate_data(dc); - dc->data = data; -} - -inline -ull DC_rbound(const DeltaChunk* dc) -{ - return dc->to + dc->ts; -} - -// Apply -inline -void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) -{ - PyObject* buffer = 0; - if (dc->data){ - buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); - } else { - buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); - } - - if (PyTuple_SetItem(tmpargs, 0, buffer)){ - assert(0); - } - - // tuple steals reference, and will take care about the deallocation - PyObject_Call(writer, tmpargs, NULL); - -} - -// Copy all data from src to dest, the data pointer will be copied too -inline -void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) -{ - dest->to = src->to; - dest->ts = src->ts; - dest->so = src->so; - dest->data_shared = 0; - dest->data = NULL; - - DC_set_data(dest, src->data, src->ts, 0); -} - -// Copy all data with the given offset and size. The source offset, as well -// as the data will be truncated accordingly -inline -void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) -{ - assert(size <= src->ts); - assert(src->to + ofs + size <= DC_rbound(src)); - - dest->to = src->to + ofs; - dest->ts = size; - dest->so = src->so + ofs; - dest->data = NULL; - - if (src->data){ - DC_set_data(dest, src->data + ofs, size, 0); - } else { - dest->data_shared = 0; - } -} - - -// DELTA CHUNK VECTOR -///////////////////// - -typedef struct { - DeltaChunk* mem; // Memory - Py_ssize_t size; // Size in DeltaChunks - Py_ssize_t reserved_size; // Reserve in DeltaChunks -} DeltaChunkVector; - - - -// Reserve enough memory to hold the given amount of delta chunks -// Return 1 on success -// NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DCVs change often, and reallocs are expensive -inline -int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) -{ - if (num_dc <= vec->reserved_size){ - return 1; - } - - if (num_dc - vec->reserved_size){ - num_dc += gDVC_grow_by; - } - -#ifdef DEBUG - bool was_null = vec->mem == NULL; -#endif - - if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); - } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); - } - - if (vec->mem == NULL){ - Py_FatalError("Could not allocate memory for append operation"); - } - - vec->reserved_size = num_dc; - -#ifdef DEBUG - const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; - if (!was_null) - format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); -#endif - - return vec->mem != NULL; -} - -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ -inline -int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) -{ - return DCV_reserve_memory(vec, vec->reserved_size + num_dc); -} - -int DCV_init(DeltaChunkVector* vec, ull initial_size) -{ - vec->mem = NULL; - vec->size = 0; - vec->reserved_size = 0; - - return DCV_grow_by(vec, initial_size); -} - -inline -ull DCV_len(const DeltaChunkVector* vec) -{ - return vec->size; -} - -inline -ull DCV_lbound(const DeltaChunkVector* vec) -{ - assert(vec->size && vec->mem); - return vec->mem->to; -} - -// Return item at index -inline -DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) -{ - assert(i < vec->size && vec->mem); - return &vec->mem[i]; -} - -// Return last item -inline -DeltaChunk* DCV_last(const DeltaChunkVector* vec) -{ - return DCV_get(vec, vec->size-1); -} - -inline -ull DCV_rbound(const DeltaChunkVector* vec) -{ - return DC_rbound(DCV_last(vec)); -} - -inline -int DCV_empty(const DeltaChunkVector* vec) -{ - return vec->size == 0; -} - -// Return end pointer of the vector -inline -const DeltaChunk* DCV_end(const DeltaChunkVector* vec) -{ - assert(!DCV_empty(vec)); - return vec->mem + vec->size; -} - -void DCV_destroy(DeltaChunkVector* vec) -{ - if (vec->mem){ -#ifdef DEBUG - fprintf(stderr, "Freeing %p\n", (void*)vec->mem); -#endif - - const DeltaChunk* end = &vec->mem[vec->size]; - DeltaChunk* i; - for(i = vec->mem; i < end; i++){ - DC_destroy(i); - } - - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - -// Reset this vector so that its existing memory can be filled again. -// Memory will be kept, but not cleaned up -inline -void DCV_forget_members(DeltaChunkVector* vec) -{ - vec->size = 0; -} - -// Reset the vector so that its size will be zero, and its members will -// have been deallocated properly. -// It will keep its memory though, and hence can be filled again -inline -void DCV_reset(DeltaChunkVector* vec) -{ - if (vec->size == 0) - return; - - DeltaChunk* dc = vec->mem; - const DeltaChunk* dcend = DCV_end(vec); - for(;dc < dcend; dc++){ - DC_destroy(dc); - } - - vec->size = 0; -} - - -// Append one chunk to the end of the list, and return a pointer to it -// It will not have been initialized ! -static inline -DeltaChunk* DCV_append(DeltaChunkVector* vec) -{ - if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDVC_grow_by); - } - - DeltaChunk* next = vec->mem + vec->size; - vec->size += 1; - return next; -} - -// Return delta chunk being closest to the given absolute offset -inline -DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) -{ - assert(vec->mem); - - ull lo = 0; - ull hi = vec->size; - ull mid; - DeltaChunk* dc; - - while (lo < hi) - { - mid = (lo + hi) / 2; - dc = vec->mem + mid; - if (dc->to > ofs){ - hi = mid; - } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { - return dc; - } else { - lo = mid + 1; - } - } - - return DCV_last(vec); -} - -// Assert the given vector has correct datachunks -// return 1 on success -int DCV_dbg_check_integrity(const DeltaChunkVector* vec) -{ - if(DCV_empty(vec)){ - return 0; - } - const DeltaChunk* i = vec->mem; - const DeltaChunk* end = DCV_end(vec); - - ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); - ull acc_size = 0; - for(; i < end; i++){ - acc_size += i->ts; - } - if (acc_size != aparent_size) - return 0; - - if (vec->size < 2){ - return 1; - } - - const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = vec->mem; i < endm1; i++){ - const DeltaChunk* n = i+1; - if (DC_rbound(i) != n->to){ - return 0; - } - } - - return 1; -} - -// Write a slice as defined by its absolute offset in bytes and its size into the given -// destination. The individual chunks written will be a deep copy of the source -// data chunks -// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort -// of append-only memory pool would improve performance -inline -void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) -{ - //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); - assert(DCV_lbound(src) <= ofs); - assert((ofs + size) <= DCV_rbound(src)); - - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); - - // partial overlap - if (cdc->to != ofs) { - DeltaChunk* destc = DCV_append(dest); - const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); - cdc += 1; - size -= destc->ts; - - if (size == 0){ - return; - } - } - - const DeltaChunk* vecend = DCV_end(src); - for( ;(cdc < vecend) && size; ++cdc) - { - if (cdc->ts < size) { - DC_copy_to(cdc, DCV_append(dest)); - size -= cdc->ts; - } else { - DC_offset_copy_to(cdc, DCV_append(dest), 0, size); - size = 0; - break; - } - } - - assert(size == 0); -} - - -// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which -// originates in to -// 'at' will be replaced by the items to insert ( special purpose ) -// 'at' will be properly destroyed, but all items will just be copied bytewise -// using memcpy. Hence from must just forget about them ! -// IMPORTANT: to must have an appropriate size already -inline -void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) -{ - //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); - assert(from->size > 1); - assert(to->size + from->size - 1 <= to->reserved_size); - - // -1 because we replace 'at' - DC_destroy(at); - - // If we are somewhere in the middle, we have to make some space - if (DCV_last(to) != at) { - //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); - memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - } - - // Finally copy all the items in - memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); - - // FINALLY: update size - to->size += from->size - 1; -} - -// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the -// caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) -{ - Py_ssize_t dci = 0; - Py_ssize_t iend = tdcv->size; - DeltaChunk* dc; - - DBG_check(tdcv); - DBG_check(bdcv); - - for (;dci < iend; dci++) - { - // Data chunks don't need processing - dc = DCV_get(tdcv, dci); - if (dc->data){ - continue; - } - - // Copy Chunk Handling - DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - DBG_check(tmpl); - assert(tmpl->size); - - // move target bounds - DeltaChunk* tdc = tmpl->mem; - DeltaChunk* tdcend = tmpl->mem + tmpl->size; - const ull ofs = dc->to - dc->so; - for(;tdc < tdcend; tdc++){ - tdc->to += ofs; - } - - // insert slice into our list - if (tmpl->size == 1){ - // Its not data, so destroy is not really required, anyhow ... - DC_destroy(dc); - *dc = *DCV_get(tmpl, 0); - } else { - DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); - dc = DCV_get(tdcv, dci); - DCV_replace_one_by_many(tmpl, tdcv, dc); - // Compensate for us being replaced - dci += tmpl->size-1; - iend += tmpl->size-1; - } - - DBG_check(tdcv); - - // make sure the members will not be deallocated by the list - DCV_forget_members(tmpl); - } -} - -// DELTA CHUNK LIST (PYTHON) -///////////////////////////// - -typedef struct { - PyObject_HEAD - // ----------- - DeltaChunkVector vec; - -} DeltaChunkList; - - -static -int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) -{ - if(args && PySequence_Size(args) > 0){ - PyErr_SetString(PyExc_ValueError, "Too many arguments"); - return -1; - } - - DCV_init(&self->vec, 0); - return 0; -} - -static -void DCL_dealloc(DeltaChunkList* self) -{ - DCV_destroy(&(self->vec)); -} - -static -PyObject* DCL_len(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); -} - -static inline -ull DCL_rbound(DeltaChunkList* self) -{ - if (DCV_empty(&self->vec)) - return 0; - return DCV_rbound(&self->vec); -} - -static -PyObject* DCL_py_rbound(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DCL_rbound(self)); -} - -// Write using a write function, taking remaining bytes from a base buffer -static -PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) -{ - PyObject* pybuf = 0; - PyObject* writeproc = 0; - if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ - PyErr_BadArgument(); - return NULL; - } - - if (!PyObject_CheckReadBuffer(pybuf)){ - PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - - if (!PyCallable_Check(writeproc)){ - PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); - return NULL; - } - - const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DCV_end(&self->vec); - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); - - PyObject* tmpargs = PyTuple_New(1); - - for(; i < end; i++){ - DC_apply(i, data, writeproc, tmpargs); - } - - Py_DECREF(tmpargs); - Py_RETURN_NONE; -} - -static PyMethodDef DCL_methods[] = { - {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, - {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject DeltaChunkListType = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "DeltaChunkList", /*tp_name*/ - sizeof(DeltaChunkList), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)DCL_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "Minimal Delta Chunk List",/* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - DCL_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -// Makes a new copy of the DeltaChunkList - you have to do everything yourselve -// in C ... want C++ !! -DeltaChunkList* DCL_new_instance(void) -{ - DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); - assert(dcl); - - DCL_init(dcl, 0, 0); - assert(dcl->vec.size == 0); - assert(dcl->vec.mem == NULL); - return dcl; -} - -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) -{ - // obtain iterator - PyObject* stream_iter = 0; - if (!PyIter_Check(dstreams)){ - stream_iter = PyObject_GetIter(dstreams); - if (!stream_iter){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); - return NULL; - } - } else { - stream_iter = dstreams; - } - - DeltaChunkVector dcv; - DeltaChunkVector tdcv; - DeltaChunkVector tmpl; - DCV_init(&dcv, 100); // should be enough to keep the average text file - DCV_init(&tdcv, 0); - DCV_init(&tmpl, 200); - - unsigned int dsi = 0; - PyObject* ds = 0; - int error = 0; - for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) - { - PyObject* db = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(db)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); - goto loop_end; - } - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* dend = data + dlen; - - // read header - const ull base_size = msb_size(&data, dend); - const ull target_size = msb_size(&data, dend); - - // estimate number of ops - assume one third adds, half two byte (size+offset) copies - const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); - DCV_reserve_memory(&dcv, approx_num_cmds); - - // parse command stream - ull tbw = 0; // Amount of target bytes written - bool is_shared_data = dsi != 0; - bool is_first_run = dsi == 0; - - assert(data < dend); - while (data < dend) - { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - const unsigned long rbound = cp_off + cp_size; - if (rbound < cp_size || - rbound > base_size){ - // this really shouldn't happen - error = 1; - assert(0); - break; - } - - DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); - tbw += cp_size; - - } else if (cmd) { - // Compression reduces fragmentation though, which is why we do it - // in all cases. - // It makes the more sense the more consecutive add-chunks we have, - // its more likely in big deltas, for big binary files - const uchar* add_start = data - 1; - const uchar* add_end = dend; - ull num_bytes = cmd; - data += cmd; - ull num_chunks = 1; - while (data < dend){ - //while (0){ - const char c = *data; - if (c & 0x80){ - add_end = data; - break; - } else { - data += 1 + c; // advance by 1 to skip add cmd - num_bytes += c; - num_chunks += 1; - } - } - - #ifdef DEBUG - assert(add_end - add_start > 0); - if (num_chunks > 1){ - fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); - } - #endif - - DeltaChunk* dc = DCV_append(&dcv); - DC_init(dc, tbw, num_bytes, 0); - - // gather the data, or (possibly) share single blocks - if (num_chunks > 1){ - uchar* dcdata = PyMem_Malloc(num_bytes); - while (add_start < add_end){ - const char bytes = *add_start++; - memcpy((void*)dcdata, (void*)add_start, bytes); - dcdata += bytes; - add_start += bytes; - } - DC_set_data_with_ownership(dc, dcdata-num_bytes); - } else { - DC_set_data(dc, data - cmd, cmd, is_shared_data); - } - - tbw += num_bytes; - } else { - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - goto loop_end; - } - }// END handle command opcodes - if (tbw != target_size){ - PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); - error = 1; - } - - if (!is_first_run){ - DCV_connect_with_base(&tdcv, &dcv, &tmpl); - } - - if (is_first_run){ - tdcv = dcv; - // wipe out dcv without destroying the members, get its own memory - DCV_init(&dcv, tdcv.size); - } else { - // destroy members, but keep memory - DCV_reset(&dcv); - } - -loop_end: - // perform cleanup - Py_DECREF(ds); - Py_DECREF(db); - - if (error){ - break; - } - }// END for each stream object - - if (dsi == 0 && ! error){ - PyErr_SetString(PyExc_ValueError, "No streams provided"); - } - - if (stream_iter != dstreams){ - Py_DECREF(stream_iter); - } - - DCV_destroy(&tmpl); - if (dsi > 1){ - // otherwise dcv equals tcl - DCV_destroy(&dcv); - } - - // Return the actual python object - its just a container - DeltaChunkList* dcl = DCL_new_instance(); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdcv would be deallocated by the chunk list - DCV_destroy(&tdcv); - error = 1; - } else { - // Plain copy, don't deallocate - dcl->vec = tdcv; - } - - if (error){ - // Will dealloc tdcv - Py_XDECREF(dcl); - return NULL; - } - - return (PyObject*)dcl; -} - static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; @@ -973,13 +93,8 @@ PyMODINIT_FUNC init_fun(void) { PyObject *m; - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - m = Py_InitModule3("_fun", py_fun, NULL); if (m == NULL) return; - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "Noddy", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index 038b4c761..e17460bf6 100644 --- a/fun.py +++ b/fun.py @@ -654,6 +654,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: # raise ImportError; # DEBUG - from _fun import connect_deltas + from _delta_apply import connect_deltas except ImportError: pass diff --git a/setup.py b/setup.py index 265156df2..7146ea833 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,10 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._fun', ['_fun.c'])], + ext_modules=[ + Extension('gitdb._fun', ['_fun.c']), + Extension('gitdb._delta_apply', ['_delta_apply.c']) + ], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 9c5672e1b0cf56f1cf5151d60acb35d610e904a8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:55:22 +0200 Subject: [PATCH 0094/3719] Now building a single module called _perf which contains all the performance enhancements, which increases loadtimes, less is more --- _delta_apply.c | 22 ---------------------- _fun.c | 12 +++++++++--- fun.py | 2 +- pack.py | 2 +- setup.py | 5 +---- 5 files changed, 12 insertions(+), 31 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 40ea60c56..d81e65d15 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -879,25 +879,3 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) return (PyObject*)dcl; } -static PyMethodDef py_fun[] = { - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, - { NULL, NULL, 0, NULL } -}; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC init_delta_apply(void) -{ - PyObject *m; - - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - - m = Py_InitModule3("_delta_apply", py_fun, NULL); - if (m == NULL) - return; - - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); -} diff --git a/_fun.c b/_fun.c index 1881bfb05..386068577 100644 --- a/_fun.c +++ b/_fun.c @@ -1,4 +1,5 @@ #include +#include "_delta_apply.c" static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -80,21 +81,26 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) Py_RETURN_NONE; } - static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ #define PyMODINIT_FUNC void #endif -PyMODINIT_FUNC init_fun(void) +PyMODINIT_FUNC init_perf(void) { PyObject *m; - m = Py_InitModule3("_fun", py_fun, NULL); + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + + m = Py_InitModule3("_perf", py_fun, NULL); if (m == NULL) return; + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index e17460bf6..0b14f82ac 100644 --- a/fun.py +++ b/fun.py @@ -654,6 +654,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: # raise ImportError; # DEBUG - from _delta_apply import connect_deltas + from _perf import connect_deltas except ImportError: pass diff --git a/pack.py b/pack.py index 91323fcce..79ffcc217 100644 --- a/pack.py +++ b/pack.py @@ -24,7 +24,7 @@ ) try: - from _fun import PackIndexFile_sha_to_index + from _perf import PackIndexFile_sha_to_index except ImportError: pass # END try c module diff --git a/setup.py b/setup.py index 7146ea833..7e50e0fe7 100755 --- a/setup.py +++ b/setup.py @@ -77,10 +77,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[ - Extension('gitdb._fun', ['_fun.c']), - Extension('gitdb._delta_apply', ['_delta_apply.c']) - ], + ext_modules=[Extension('gitdb._perf', ['_fun.c'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From fa03f746746df99763deb45943988d68fb450b4b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 15:51:04 +0200 Subject: [PATCH 0095/3719] Reverse Delta Application was a nice experiment, as it has one major flaw: Currently it integrates chunks from its base into the topmost delta chunk list, which causes plenty of mem-move operations. Plenty means, many many many, and its getting worse the more deltas you have of course. The algorithm was supposed to reduce the amount of memory activity, but failed at this point, making it worse than before. Probably it would just be fastest to implement the previous python algorithm, which swaps two buffers, in c --- _delta_apply.c | 11 +++++++---- stream.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index d81e65d15..e667a2eda 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -398,7 +398,6 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) inline void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { - //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); assert(DCV_lbound(src) <= ofs); assert((ofs + size) <= DCV_rbound(src)); @@ -443,7 +442,6 @@ void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull inline void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { - //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); assert(to->size + from->size - 1 <= to->reserved_size); @@ -452,7 +450,6 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } @@ -725,7 +722,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const ull target_size = msb_size(&data, dend); // estimate number of ops - assume one third adds, half two byte (size+offset) copies - const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + // Assume good compression for the adds + const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); DCV_reserve_memory(&dcv, approx_num_cmds); // parse command stream @@ -825,6 +823,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DCV_connect_with_base(&tdcv, &dcv, &tmpl); } + #ifdef DEBUG + fprintf(stderr, "tdcv->size = %i, tdcv->reserved_size = %i\n", (int)tdcv.size, (int)tdcv.reserved_size); + fprintf(stderr, "dcv->size = %i, dcv->reserved_size = %i\n", (int)dcv.size, (int)dcv.reserved_size); + #endif + if (is_first_run){ tdcv = dcv; // wipe out dcv without destroying the members, get its own memory diff --git a/stream.py b/stream.py index af4591f85..8f9f9c386 100644 --- a/stream.py +++ b/stream.py @@ -337,6 +337,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. + # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) # call len directly, as the (optional) c version doesn't implement the sequence From ff9c83a3fea26653d725686a692d0100efb63382 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 16:28:07 +0200 Subject: [PATCH 0096/3719] Disabled new implementation in favor of the old one - all that's needed is a c implementation of apply delta data --- _delta_apply.c | 4 ++++ stream.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index e667a2eda..51f55e892 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -450,6 +450,10 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { + // IMPORTANT: This memmove kills the performance in case of large deltas + // Causing everything to slow down enormously. Its logical, as the memory + // gets shifted each time we insert nodes, for each chunk, for ever smaller + // chunks depending on the deltas memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } diff --git a/stream.py b/stream.py index 8f9f9c386..2c24426c6 100644 --- a/stream.py +++ b/stream.py @@ -325,7 +325,7 @@ def __init__(self, stream_list): self._dstreams = tuple(stream_list[:-1]) self._br = 0 - def _set_cache_(self, attr): + def _set_cache_too_slow(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python, every function call costs @@ -360,7 +360,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_brute_(self, attr): + def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From 95d820239d5444d59ae67f50de715f1574c7213b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 17:38:53 +0200 Subject: [PATCH 0097/3719] apply_delta now has a C implementation, which is only 25 percent faster for small files ( the overhead of all the rest is very high ), but 4 times faster for large files, where the enormous call overhead coming in with python really starts to show --- _delta_apply.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ _fun.c | 3 ++- stream.py | 17 ++++++------ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 51f55e892..dfb0a94c3 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -886,3 +886,73 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) return (PyObject*)dcl; } + +// Write using a write function, taking remaining bytes from a base buffer +// replaces the corresponding method in python +static +PyObject* apply_delta(PyObject* self, PyObject* args) +{ + PyObject* pybbuf = 0; + PyObject* pydbuf = 0; + PyObject* pytbuf = 0; + if (!PyArg_ParseTuple(args, "OOO", &pybbuf, &pydbuf, &pytbuf)){ + PyErr_BadArgument(); + return NULL; + } + + PyObject* objects[] = { pybbuf, pydbuf, pytbuf }; + assert(sizeof(objects) / sizeof(PyObject*) == 3); + + uint i; + for(i = 0; i < 3; i++){ + if (!PyObject_CheckReadBuffer(objects[i])){ + PyErr_SetString(PyExc_ValueError, "Argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + } + + Py_ssize_t lbbuf; Py_ssize_t ldbuf; Py_ssize_t ltbuf; + const uchar* bbuf; const uchar* dbuf; + uchar* tbuf; + PyObject_AsReadBuffer(pybbuf, (const void**)(&bbuf), &lbbuf); + PyObject_AsReadBuffer(pydbuf, (const void**)(&dbuf), &ldbuf); + + if (PyObject_AsWriteBuffer(pytbuf, (void**)(&tbuf), <buf)){ + PyErr_SetString(PyExc_ValueError, "Argument 3 must be a writable buffer"); + return NULL; + } + + const uchar* data = dbuf; + const uchar* dend = dbuf + ldbuf; + + while (data < dend) + { + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + memcpy(tbuf, bbuf + cp_off, cp_size); + tbuf += cp_size; + + } else if (cmd) { + memcpy(tbuf, data, cmd); + tbuf += cmd; + data += cmd; + } else { + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + return NULL; + } + }// END handle command opcodes + + Py_RETURN_NONE; +} diff --git a/_fun.c b/_fun.c index 386068577..befee4ec4 100644 --- a/_fun.c +++ b/_fun.c @@ -83,7 +83,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "See python implementation" }, + { "apply_delta", (PyCFunction)apply_delta, METH_VARARGS, "See python implementation" }, { NULL, NULL, 0, NULL } }; diff --git a/stream.py b/stream.py index 2c24426c6..cedb70cf3 100644 --- a/stream.py +++ b/stream.py @@ -22,6 +22,11 @@ zlib ) +try: + from _perf import apply_delta as c_apply_delta +except ImportError: + pass + __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') @@ -413,7 +418,10 @@ def _set_cache_(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf); + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### # finally, swap out source and target buffers. The target is now the @@ -430,13 +438,6 @@ def _set_cache_(self, attr): self._mm_target = bbuf self._size = final_target_size - # TODO: Once that works, figure out the ordering of the opcodes. If they - # are always in-order/sequential, an alternate implementation could - # use stream access only. Of course this would mean we would read - # all deltas in advance, analyse the opcode ranges to determine a final - # concatenated opcode list which indicates what to copy from which delta - # to which position. This preprocessing would allow true streaming - def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: From f3284c1edf3670dca0cf7fb11ad50a36fe53e782 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 19:21:48 +0200 Subject: [PATCH 0098/3719] Integrated new algorithm into the stream class, it will now be chosen depending on the context to figure out which one to use. For some reason, the c version that is slow for big files really rocks when its about small files. Its better than the respective c implementation of the normal delta apply --- stream.py | 23 ++++++++++++++++++++++- test/performance/test_pack.py | 33 +++++++++++---------------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/stream.py b/stream.py index cedb70cf3..f5e05e114 100644 --- a/stream.py +++ b/stream.py @@ -22,8 +22,10 @@ zlib ) +has_perf_mod = False try: from _perf import apply_delta as c_apply_delta + has_perf_mod = True except ImportError: pass @@ -330,7 +332,7 @@ def __init__(self, stream_list): self._dstreams = tuple(stream_list[:-1]) self._br = 0 - def _set_cache_too_slow(self, attr): + def _set_cache_too_slow_without_c(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python, every function call costs @@ -366,6 +368,15 @@ def _set_cache_too_slow(self, attr): self._mm_target.seek(0) def _set_cache_(self, attr): + """Determine which version to use depending on the configuration of the deltas + :note: we are only called if we have the performance module""" + # otherwise it depends on the amount of memory to shift around + if len(self._dstreams) > 1 and self._bstream.size < 150000: + return self._set_cache_too_slow_without_c(attr) + else: + return self._set_cache_brute_(attr) + + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() @@ -438,6 +449,13 @@ def _set_cache_(self, attr): self._mm_target = bbuf self._size = final_target_size + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + + #} END configuration + def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: @@ -654,4 +672,7 @@ def close(self): def write(self, data): return len(data) + #} END W streams + + diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index af468b0be..f9169ffb0 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -25,29 +25,18 @@ def test_pack_random_access(self): # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info - access_times = list() - for rand in range(2): - if rand: - random.shuffle(sha_list) - # END shuffle shas - st = time() - for sha in sha_list: - pdb_pack_info(sha) - # END for each sha to look up - elapsed = time() - st - access_times.append(elapsed) - - # discard cache - del(pdb._entities) - pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs (random=%i) in %f s ( %f shas/s )" % (ns, len(pdb.entities()), rand, elapsed, ns / elapsed) - # END for each random mode - elapsed_order, elapsed_rand = access_times - - # well, its never really sequencial regarding the memory patterns, but it - # shows how well the prioriy cache performs - print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100)) + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + # discard cache + del(pdb._entities) + pdb.entities() + print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + # END for each random mode # query info and streams only max_items = 10000 # can wait longer when testing memory From f59c58d9478ddc49964b5bcd0711d06a957e4680 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 21:46:05 +0200 Subject: [PATCH 0099/3719] Added new stream type which will request its size from its stream. This triggers full decompression, currently, but allows work to be delayed even further.If people want a stream, they usually read it anyway. Then it doesn't matter which attriubute they query first --- base.py | 17 +++++++++++++++++ pack.py | 8 ++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/base.py b/base.py index 938a09242..d0bcc0866 100644 --- a/base.py +++ b/base.py @@ -135,6 +135,23 @@ def read(self, size=-1): @property def stream(self): return self[3] + + #} END stream reader interface + + +class ODeltaStream(OStream): + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + #} END stream reader interface diff --git a/pack.py b/pack.py index 79ffcc217..30da52c63 100644 --- a/pack.py +++ b/pack.py @@ -34,6 +34,7 @@ OStream, OPackInfo, OPackStream, + ODeltaStream, ODeltaPackInfo, ODeltaPackStream, ) @@ -584,14 +585,9 @@ def _object(self, sha, as_stream, index=-1): # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) - buf = streams[0].read(512) - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - - streams[0].stream.seek(0) # assure it can be read by the delta reader dstream = DeltaApplyReader.new(streams) - return OStream(sha, dstream.type, target_size, dstream) + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) From dcdd0fd9aa8aea6989cbf1530e73c6c86fc95761 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 19:17:22 +0200 Subject: [PATCH 0100/3719] Added initial version of a document to show possible ways to stream delta data most efficiently --- doc/source/algorithm.rst | 101 +++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 1 + stream.py | 3 ++ 3 files changed, 105 insertions(+) create mode 100644 doc/source/algorithm.rst diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst new file mode 100644 index 000000000..d1e4a9ba1 --- /dev/null +++ b/doc/source/algorithm.rst @@ -0,0 +1,101 @@ +######################## +Discussion of Algorithms +######################## + +************ +Introduction +************ +As you know, the pure-python object database support for GitPython is provided by the GitDB project. It is meant to be my backup plan to ensure that the DataVault (http://www.youtube.com/user/ByronBates99?feature=mhum#p/c/2A5C6EF5BDA8DB5C ) can handle reading huge files, especially those which were consolidated into packs. A nearly fully packed state is anticipated for the data-vaults repository, and reading these packs efficiently is an essential task. + +This document informs you about my findings in the struggle to improve the way packs are read to reduce memory usage required to handle huge files. It will try to conclude where future development could go to assure big delta-packed files can be read without the need of 8GB+ RAM. + +GitDB's main feature is the use of streams, hence the amount of memory used to read a database object is minimized, at the cost of some additional processing overhead to keep track of the stream state. This works great for legacy objects, which are essentially a zip-compressed byte-stream. + +Streaming data from delta-packed objects is far more difficult though, and only technically possible within certain limits, and at relatively high processing costs. My first observation was that there doesn't appear to be 'the one and only' algorithm which is generally superior. They all have their pros and cons, but fortunately this allows the implementation to choose the one most suited based on the amount of delta streams, as well as the size of the base, which allows an early and cheap estimate of the target size of the final data. + +********************************** +Traditional Delta-Apply-Algorithms +********************************** + +The brute-force CGit delta-apply algorithm +========================================== +CGit employs a simple and relatively brute-force algorithm, which resolves all delta streams recursively. When the recursion reaches the base level of the deltas, it will be decompressed into a buffer, then the first delta gets decompressed into a second buffer. From that, the target size of the delta can be extracted, to allocated a third buffer to hold the result of the operation, which consists of reading the delta stream byte-wise, to apply the operations in order, as described by single-byte opcodes. During recursion, each target buffer of the preceding delta-apply operation is used as base buffer for the next delta-apply operation, until the last delta was applied, leaving the final target buffer as result. + +About Delta-Opcodes +------------------- +There are only two kinds of opcodes, 'add-bytes' and 'copy-from-base'. One 'add-bytes' opcode can encode up to 7 bit of additional bytes to be copied from the delta stream into the target buffer. +A 'copy-from-base' opcode encodes a 32 bit offset into the base buffer, as well as the amount of bytes to be copied, which are up to 2^24 bytes. We may conclude that delta-bases may not be larger than 2^32+2^24 bytes in the current, extensible, implementation. +When generating the delta, git prefers copy operations over add operations, as they are much more efficient. Usually, the most recent, or biggest version of a file is used as base, whereas older and smaller versions of the file are expressed by copying only portions of the newest file. As it is not efficiently possible to represent all changes that way, add-bytes operations fill the gap where needed. All this explains why git can only add 128 bytes with one opcode, as it tries to minimize their use. +This implies that recent file history can usually be extracted faster than old history, which may involve many more deltas. + +Performance considerations +-------------------------- +The performance bottleneck of this algorithm appear to be the throughput of your RAM, as both opcodes will just trigger memcpy operations from one memory location to another, times the amount of deltas to apply. This in fact is very fast, even for big files above 100 MB. +Memory allocation could become an issue as you need the base buffer, the target buffer as well as the decompressed delta stream in memory at the same time. The continuous allocation and deallocation of possibly big buffers may support memory fragmentation. Whether it really kicks in, especially on 64 bit machines, is unproven though. +Nonetheless, the cgit implementation is currently the fastest one. + +The brute-force GitDB algorithm +=============================== +Despite of working essentially the same way as the CGit brute-force algorithm, GitDB minimizes the amount of allocations to 2 + num of deltas. The amount memory allocated peaks whiles the deltas are applied, as the base and target buffer, as well as the decompressed stream, are held in memory. +To achieve this, GitDB retrieves all delta-streams in advance, and peaks into their header information to determine the maximum size of the target buffer, just by reading 512 bytes of the compressed stream. If there is more than one delta to apply, the base buffer is set large enough to hold the biggest target buffer required by the delta streams. +Now it is possible to iterate all deltas, oldest first, newest last, and apply them using the buffers. At the end of each iteration, the buffers are swapped. + +Performance Results +------------------- +The performance test is performed on an aggressively packed repository with the history of cgit. 5000 sha's are extracted and read one after another. The delta-chains have a length of 0 to about 35. The pure-python implementation can stream the data of all objects (totaling 62,2 MiB) with an average rate of 8.1 MiB/s, which equals about 654 streams/s. +There are two bottlenecks: The major is the collection of the delta streams, which involves plenty of pack-lookup. This lookup is expensive in python, and is overly expensive. Its not overly critical though, as it only limits the amount of streams per second, not the actual data rate when applying the deltas. +Applying the deltas happens to be the second bottleneck, if the files to be processed get bigger. The more opcodes have to be processed, the more python slow function calls will dominate the result. As an example, it takes nearly 8 seconds to unpack a 125 MB file, where cgit only takes 2.4 s. + +To eliminate a few performance concerns, some key routines were rewritten in C. This changes the numbers of this particular test significantly, but not drastically, as the major bottleneck (delta collection) is still in place. Another performance reduction is due to the fact that plenty of other code related to the deltas is still pure-python. +Now all 5000 objects can be read at a rate of 11.1 MiB /s, or 892 streams/s. Fortunately, unpacking a big object is now done in 2.5s, which is just a tad slower than cgit, but with possibly less memory fragmentation. + + +************************************** +Paving the way towards delta streaming +************************************** + +GitDB's reverse delta aggregation algorithm +=========================================== +The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. + +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**), which are kept in vectors. Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. + +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream. + +The merging works by following a set of rules: + * Merge into the top-level delta from the youngest ancestor delta to the oldest one + * When merging one ADS, iterate from the first to the last chunk in TDS, then: + + * skip all add-bytes DCs. If bytes are added, these will always overwrite any operation coming from any ADS at the same offset. + * copy-from-base DCs will copy a slice of the respective portion of the ADS ( as defined by their base offset ) and use it to replace the original chunk. This acts as a 'virtual' copy-from-base operation. + + * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. + +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. + +The memory consumption during the TDS processing are the uncompressed delta-bytes, the parsed DS, as well as the TDS. Afterwards one requires an allocated base buffer, the target buffer, as well as the TDS. +It is clearly visible that the current implementation does not at all reduce memory consumption, but the opposite is true as the TDS can be large for large files. + +Performance Results +------------------- +The benchmarking context was the same as for the brute-force GitDB algorithm. This implementation is far more complex than the said brute-force implementation, which clearly reflects in the numbers. It's pure-python throughput is at only 1.1 MiB/s, which equals 89 streams/s. +The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. + +To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. +The throughput reaches 15.6 MiB/s, which equals 1267 streams/s, which makes it 14 times faster than the pure python version, and amazingly even 1.4 times faster than the brute-force C implementation. + +*TODO* +All this comes at a relatively high memory consumption, and heavily degrading performance with raising file sizes. A 125 MB file took 8 seconds to unpack for instance. The reason for this is the current implementation's brute-force algorithm to insert ADS slices into the TDS, which triggers an enormous amount of memmove operations of large portions of overlapping memory portions. + +Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. + + +Future work +=========== +* Analyse TDS to determine which sections from base buffer need to be copied. Read these in order of lowest to highest offset from base stream, and copy them into a smaller memory map. Relink source offsets to point to the new location in the new buffer. +* recompress TDS into bytestream to minimize the memory footprint when streaming, allocate the stream into a memory map. + +With that in place, streaming becomes trivial. Memory consumption + + diff --git a/doc/source/index.rst b/doc/source/index.rst index d414cb22f..5223e6bd9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: intro tutorial api + algorithm changes Indices and tables diff --git a/stream.py b/stream.py index f5e05e114..38c86dae7 100644 --- a/stream.py +++ b/stream.py @@ -379,6 +379,9 @@ def _set_cache_(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: From 6656c66f7edcf746132ce52d9ed714283140eaa0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 20:39:20 +0200 Subject: [PATCH 0101/3719] Implemented simple pre-pass to count offsets to help calculate where each chunk is going to be. This way, memmove becomes memcpy, and only one grow is required --- _delta_apply.c | 189 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 129 insertions(+), 60 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index dfb0a94c3..76d482b0d 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -11,7 +11,7 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDVC_grow_by = 100; +const ull gDCV_grow_by = 100; #ifdef DEBUG #define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) @@ -170,8 +170,8 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) return 1; } - if (num_dc - vec->reserved_size){ - num_dc += gDVC_grow_by; + if (num_dc - vec->reserved_size < 10){ + num_dc += gDCV_grow_by; } #ifdef DEBUG @@ -255,6 +255,12 @@ ull DCV_rbound(const DeltaChunkVector* vec) return DC_rbound(DCV_last(vec)); } +inline +ull DCV_size(const DeltaChunkVector* vec) +{ + return DCV_rbound(vec) - DCV_lbound(vec); +} + inline int DCV_empty(const DeltaChunkVector* vec) { @@ -269,6 +275,14 @@ const DeltaChunk* DCV_end(const DeltaChunkVector* vec) return vec->mem + vec->size; } +// return first item in vector +inline +DeltaChunk* DCV_first(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return vec->mem; +} + void DCV_destroy(DeltaChunkVector* vec) { if (vec->mem){ @@ -306,7 +320,7 @@ void DCV_reset(DeltaChunkVector* vec) if (vec->size == 0) return; - DeltaChunk* dc = vec->mem; + DeltaChunk* dc = DCV_first(vec); const DeltaChunk* dcend = DCV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); @@ -322,7 +336,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDVC_grow_by); + DCV_grow_by(vec, gDCV_grow_by); } DeltaChunk* next = vec->mem + vec->size; @@ -364,7 +378,7 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) if(DCV_empty(vec)){ return 0; } - const DeltaChunk* i = vec->mem; + const DeltaChunk* i = DCV_first(vec); const DeltaChunk* end = DCV_end(vec); ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); @@ -380,7 +394,7 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) } const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = vec->mem; i < endm1; i++){ + for(i = DCV_first(vec); i < endm1; i++){ const DeltaChunk* n = i+1; if (DC_rbound(i) != n->to){ return 0; @@ -390,46 +404,82 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) return 1; } +// Return the amount of chunks a slice at the given spot would have +inline +uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) +{ + uint num_dc = 0; + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + const ull relofs = ofs - cdc->to; + size -= cdc->ts - relofs < size ? cdc->ts - relofs : size; + num_dc += 1; + cdc += 1; + + if (size == 0){ + return num_dc; + } + } + + const DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc){ + num_dc += 1; + if (cdc->ts < size) { + size -= cdc->ts; + } else { + size = 0; + break; + } + } + + return num_dc; +} + // Write a slice as defined by its absolute offset in bytes and its size into the given -// destination. The individual chunks written will be a deep copy of the source +// destination memory. The individual chunks written will be a deep copy of the source // data chunks -// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort -// of append-only memory pool would improve performance +// Return: number of chunks in the slice inline -void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, ull size) { assert(DCV_lbound(src) <= ofs); assert((ofs + size) <= DCV_rbound(src)); DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + uint num_chunks = 0; // partial overlap if (cdc->to != ofs) { - DeltaChunk* destc = DCV_append(dest); const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); cdc += 1; - size -= destc->ts; + size -= dest->ts; + dest += 1; + num_chunks += 1; if (size == 0){ - return; + return num_chunks; } } const DeltaChunk* vecend = DCV_end(src); for( ;(cdc < vecend) && size; ++cdc) { + num_chunks += 1; if (cdc->ts < size) { - DC_copy_to(cdc, DCV_append(dest)); + DC_copy_to(cdc, dest++); size -= cdc->ts; } else { - DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + DC_offset_copy_to(cdc, dest++, 0, size); size = 0; break; } } assert(size == 0); + return num_chunks; } @@ -458,64 +508,85 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, } // Finally copy all the items in - memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + memcpy((void*) at, (void*)DCV_first(from), from->size*sizeof(DeltaChunk)); // FINALLY: update size to->size += from->size - 1; } // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the -// caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +// delta to apply. +bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) { - Py_ssize_t dci = 0; - Py_ssize_t iend = tdcv->size; - DeltaChunk* dc; - DBG_check(tdcv); DBG_check(bdcv); - for (;dci < iend; dci++) + uint* offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + if (!offset_array){ + return 0; + } + + fprintf(stderr, "old size = %i\n", (int)tdcv->size); + uint* pofs = offset_array; + uint num_addchunks = 0; + + DeltaChunk* dc = DCV_first(tdcv); + const DeltaChunk* dcend = DCV_end(tdcv); + const ull oldsize = DCV_size(tdcv); + + // OFFSET RUN + for (;dc < dcend; dc++, pofs++) { // Data chunks don't need processing - dc = DCV_get(tdcv, dci); + *pofs = num_addchunks; if (dc->data){ continue; } - // Copy Chunk Handling - DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - DBG_check(tmpl); - assert(tmpl->size); - - // move target bounds - DeltaChunk* tdc = tmpl->mem; - DeltaChunk* tdcend = tmpl->mem + tmpl->size; - const ull ofs = dc->to - dc->so; - for(;tdc < tdcend; tdc++){ - tdc->to += ofs; + // offset the next chunk by the amount of chunks in the slice + // - 1, because we replace our own chunk + num_addchunks += DCV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + } + + // reserve enough memory to hold all the new chunks + // reinit pointers, array could have been reallocated + DCV_reserve_memory(tdcv, tdcv->size + num_addchunks); + dc = DCV_last(tdcv); + dcend = DCV_first(tdcv) - 1; + + // now, that we have our pointers with the old size + tdcv->size += num_addchunks; + + // Insert slices, from the end to the beginning, which allows memcpy + // to be used, with a little help of the offset array + for (pofs -= 1; dc > dcend; dc--, pofs-- ) + { + // Data chunks don't need processing + const uint ofs = *pofs; + if (dc->data){ + // TODO: peak the preceeding chunks to figure out whether they are + // all just moved by ofs. In that case, they can move as a whole! + // just copy the chunk according to its offset + if (ofs){ + memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); + } + continue; } - // insert slice into our list - if (tmpl->size == 1){ - // Its not data, so destroy is not really required, anyhow ... - DC_destroy(dc); - *dc = *DCV_get(tmpl, 0); - } else { - DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); - dc = DCV_get(tdcv, dci); - DCV_replace_one_by_many(tmpl, tdcv, dc); - // Compensate for us being replaced - dci += tmpl->size-1; - iend += tmpl->size-1; + // Copy Chunks, and move their target offset into place + DeltaChunk* tdc = dc + ofs; + DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); + const ull relofs = dc->to - dc->so; + for(;tdc < tdcend; tdc++){ + tdc->to += relofs; } - - DBG_check(tdcv); - - // make sure the members will not be deallocated by the list - DCV_forget_members(tmpl); } + + fprintf(stderr, "NEW size = %i\n", (int)tdcv->size); + DBG_check(tdcv); + assert(DCV_size(tdcv) == oldsize); + PyMem_Free(offset_array); + return 1; } // DELTA CHUNK LIST (PYTHON) @@ -699,10 +770,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkVector dcv; DeltaChunkVector tdcv; - DeltaChunkVector tmpl; DCV_init(&dcv, 100); // should be enough to keep the average text file DCV_init(&tdcv, 0); - DCV_init(&tmpl, 200); unsigned int dsi = 0; PyObject* ds = 0; @@ -725,7 +794,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const ull base_size = msb_size(&data, dend); const ull target_size = msb_size(&data, dend); - // estimate number of ops - assume one third adds, half two byte (size+offset) copies // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); DCV_reserve_memory(&dcv, approx_num_cmds); @@ -824,7 +892,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } if (!is_first_run){ - DCV_connect_with_base(&tdcv, &dcv, &tmpl); + if (!DCV_connect_with_base(&tdcv, &dcv)){ + error = 1; + } } #ifdef DEBUG @@ -859,7 +929,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - DCV_destroy(&tmpl); if (dsi > 1){ // otherwise dcv equals tcl DCV_destroy(&dcv); From c03a46bea58d9b108cb314f9a1f0c422c05bb3bf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 21:07:21 +0200 Subject: [PATCH 0102/3719] Fixed tiny little bug that would cause our own chunk to be overridden before we make one last computation with its unaltered values --- _delta_apply.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 76d482b0d..4cdb654f2 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -456,7 +456,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); cdc += 1; size -= dest->ts; - dest += 1; + dest += 1; // must be here, we are reading the size ! num_chunks += 1; if (size == 0){ @@ -472,7 +472,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u DC_copy_to(cdc, dest++); size -= cdc->ts; } else { - DC_offset_copy_to(cdc, dest++, 0, size); + DC_offset_copy_to(cdc, dest, 0, size); size = 0; break; } @@ -526,7 +526,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) return 0; } - fprintf(stderr, "old size = %i\n", (int)tdcv->size); uint* pofs = offset_array; uint num_addchunks = 0; @@ -574,17 +573,19 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) } // Copy Chunks, and move their target offset into place + // As we could override dc when slicing, we get the data here + const ull relofs = dc->to - dc->so; + DeltaChunk* tdc = dc + ofs; DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); - const ull relofs = dc->to - dc->so; for(;tdc < tdcend; tdc++){ tdc->to += relofs; } } - fprintf(stderr, "NEW size = %i\n", (int)tdcv->size); DBG_check(tdcv); assert(DCV_size(tdcv) == oldsize); + PyMem_Free(offset_array); return 1; } From 65c9abfde0eae317c6c6dcf91918258e5e6ca33c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 21:38:48 +0200 Subject: [PATCH 0103/3719] removed some debug code --- _delta_apply.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 4cdb654f2..8cfb3f2e8 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -521,7 +521,7 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) DBG_check(tdcv); DBG_check(bdcv); - uint* offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; } @@ -531,7 +531,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) DeltaChunk* dc = DCV_first(tdcv); const DeltaChunk* dcend = DCV_end(tdcv); - const ull oldsize = DCV_size(tdcv); // OFFSET RUN for (;dc < dcend; dc++, pofs++) @@ -563,9 +562,10 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // Data chunks don't need processing const uint ofs = *pofs; if (dc->data){ - // TODO: peak the preceeding chunks to figure out whether they are + // NOTE: could peek the preceeding chunks to figure out whether they are // all just moved by ofs. In that case, they can move as a whole! - // just copy the chunk according to its offset + // tests showed that this is very rare though, even in huge deltas, so its + // not worth the extra effort if (ofs){ memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); } @@ -584,7 +584,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) } DBG_check(tdcv); - assert(DCV_size(tdcv) == oldsize); PyMem_Free(offset_array); return 1; From 78665b13ff4125f4ce3e5311d040c027bdc92a9a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 22:59:19 +0200 Subject: [PATCH 0104/3719] Updated draft with latest data, finished it by defining future ways to improve the algorithm --- doc/source/algorithm.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index d1e4a9ba1..55207b67b 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -83,19 +83,17 @@ The benchmarking context was the same as for the brute-force GitDB algorithm. Th The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. -The throughput reaches 15.6 MiB/s, which equals 1267 streams/s, which makes it 14 times faster than the pure python version, and amazingly even 1.4 times faster than the brute-force C implementation. +The throughput reaches 16.7 MiB/s, which equals 1344 streams/s, which makes it 15 times faster than the pure python version, and amazingly even 1.5 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. -*TODO* -All this comes at a relatively high memory consumption, and heavily degrading performance with raising file sizes. A 125 MB file took 8 seconds to unpack for instance. The reason for this is the current implementation's brute-force algorithm to insert ADS slices into the TDS, which triggers an enormous amount of memmove operations of large portions of overlapping memory portions. +All this comes at a relatively high memory consumption.Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. -Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. +A 125 MB file took 3.1 seconds to unpack for instance, which is only 33% slower than the c implementation of the brute-force algorithm. Future work =========== -* Analyse TDS to determine which sections from base buffer need to be copied. Read these in order of lowest to highest offset from base stream, and copy them into a smaller memory map. Relink source offsets to point to the new location in the new buffer. -* recompress TDS into bytestream to minimize the memory footprint when streaming, allocate the stream into a memory map. - -With that in place, streaming becomes trivial. Memory consumption +The current implementation of the reverse delta aggregation algorithm is already working well and fast, but leaves room for improvement in the realm of its memory consumption. One way to considerably reduce it would be to index the delta stream to determine bounds, instead of parsing it into a separate data structure +Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. From ffafb2e998220344f4674b25b6760254b2ec453e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2010 16:04:17 +0200 Subject: [PATCH 0105/3719] First adjustment to prepare the algorithm to work on deltastreams directly, without an intermediate conversion into far-too-large DeltaChunks. The new style just uses a single index, using one-fourth of the memory --- _delta_apply.c | 329 ++++++++++++++----------------------------------- 1 file changed, 93 insertions(+), 236 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 8cfb3f2e8..9768b5264 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -11,82 +11,38 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDCV_grow_by = 100; +const ull gDIV_grow_by = 100; + + +// DELTA INFO +///////////// +typedef struct { + uint dso; // delta stream offset + uint to; // target offset (cache) +} DeltaInfo; -#ifdef DEBUG -#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) -#else -#define DBG_check(vec) -#endif // DELTA CHUNK //////////////// // Internal Delta Chunk Objects +// They are just used to keep information parsed from a stream +// The data pointer is always shared typedef struct { ull to; ull ts; ull so; const uchar* data; - bool data_shared; } DeltaChunk; inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) { dc->to = to; dc->ts = ts; dc->so = so; dc->data = NULL; - dc->data_shared = 0; -} - -inline -void DC_deallocate_data(DeltaChunk* dc) -{ - if (!dc->data_shared && dc->data){ - PyMem_Free((void*)dc->data); - } - dc->data = NULL; } -inline -void DC_destroy(DeltaChunk* dc) -{ - DC_deallocate_data(dc); -} - -// Store a copy of data in our instance. If shared is 1, the data will be shared, -// hence it will only be stored, but the memory will not be touched, or copied. -inline -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) -{ - DC_deallocate_data(dc); - - if (data == 0){ - dc->data = NULL; - dc->data_shared = 0; - return; - } - - dc->data_shared = shared; - if (shared){ - dc->data = data; - } else { - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy((void*)dc->data, (void*)data, dlen); - } - -} - -// Make the given data our own. It is assumed to have the size stored in our instance -// and will be managed by us. -inline -void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) -{ - assert(data); - DC_deallocate_data(dc); - dc->data = data; -} inline ull DC_rbound(const DeltaChunk* dc) @@ -94,7 +50,8 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } -// Apply +// Apply +// TODO: remove, just left it for reference inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) { @@ -114,48 +71,16 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec } -// Copy all data from src to dest, the data pointer will be copied too -inline -void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) -{ - dest->to = src->to; - dest->ts = src->ts; - dest->so = src->so; - dest->data_shared = 0; - dest->data = NULL; - - DC_set_data(dest, src->data, src->ts, 0); -} - -// Copy all data with the given offset and size. The source offset, as well -// as the data will be truncated accordingly -inline -void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) -{ - assert(size <= src->ts); - assert(src->to + ofs + size <= DC_rbound(src)); - - dest->to = src->to + ofs; - dest->ts = size; - dest->so = src->so + ofs; - dest->data = NULL; - - if (src->data){ - DC_set_data(dest, src->data + ofs, size, 0); - } else { - dest->data_shared = 0; - } -} - // DELTA CHUNK VECTOR ///////////////////// typedef struct { - DeltaChunk* mem; // Memory - Py_ssize_t size; // Size in DeltaChunks - Py_ssize_t reserved_size; // Reserve in DeltaChunks -} DeltaChunkVector; + DeltaInfo* mem; // Memory + const uchar* dstream; // pointer to delta stream we index + Py_ssize_t size; // Size in DeltaInfos + Py_ssize_t reserved_size; // Reserve in DeltaInfos +} DeltaInfoVector; @@ -164,14 +89,14 @@ typedef struct { // NOTE: added a minimum allocation to assure reallocation is not done // just for a single additional entry. DCVs change often, and reallocs are expensive inline -int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) +int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) { if (num_dc <= vec->reserved_size){ return 1; } if (num_dc - vec->reserved_size < 10){ - num_dc += gDCV_grow_by; + num_dc += gDIV_grow_by; } #ifdef DEBUG @@ -179,9 +104,9 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) #endif if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaInfo)); } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaInfo)); } if (vec->mem == NULL){ @@ -194,7 +119,7 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; if (!was_null) format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaInfo)), vec->mem, (int)vec->reserved_size); #endif return vec->mem != NULL; @@ -207,28 +132,28 @@ large enough. Return 1 on success, 0 on failure */ inline -int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) { - return DCV_reserve_memory(vec, vec->reserved_size + num_dc); + return DIV_reserve_memory(vec, vec->reserved_size + num_dc); } -int DCV_init(DeltaChunkVector* vec, ull initial_size) +int DIV_init(DeltaInfoVector* vec, ull initial_size) { vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; - return DCV_grow_by(vec, initial_size); + return DIV_grow_by(vec, initial_size); } inline -ull DCV_len(const DeltaChunkVector* vec) +ull DIV_len(const DeltaInfoVector* vec) { return vec->size; } inline -ull DCV_lbound(const DeltaChunkVector* vec) +ull DIV_lbound(const DeltaInfoVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -236,7 +161,7 @@ ull DCV_lbound(const DeltaChunkVector* vec) // Return item at index inline -DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) +DeltaInfo* DIV_get(const DeltaInfoVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; @@ -244,54 +169,54 @@ DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) // Return last item inline -DeltaChunk* DCV_last(const DeltaChunkVector* vec) +DeltaInfo* DIV_last(const DeltaInfoVector* vec) { - return DCV_get(vec, vec->size-1); + return DIV_get(vec, vec->size-1); } inline -ull DCV_rbound(const DeltaChunkVector* vec) +ull DIV_rbound(const DeltaInfoVector* vec) { - return DC_rbound(DCV_last(vec)); + return DC_rbound(DIV_last(vec)); } inline -ull DCV_size(const DeltaChunkVector* vec) +ull DIV_size(const DeltaInfoVector* vec) { - return DCV_rbound(vec) - DCV_lbound(vec); + return DIV_rbound(vec) - DIV_lbound(vec); } inline -int DCV_empty(const DeltaChunkVector* vec) +int DIV_empty(const DeltaInfoVector* vec) { return vec->size == 0; } // Return end pointer of the vector inline -const DeltaChunk* DCV_end(const DeltaChunkVector* vec) +const DeltaInfo* DIV_end(const DeltaInfoVector* vec) { - assert(!DCV_empty(vec)); + assert(!DIV_empty(vec)); return vec->mem + vec->size; } // return first item in vector inline -DeltaChunk* DCV_first(const DeltaChunkVector* vec) +DeltaInfo* DIV_first(const DeltaInfoVector* vec) { - assert(!DCV_empty(vec)); + assert(!DIV_empty(vec)); return vec->mem; } -void DCV_destroy(DeltaChunkVector* vec) +void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG fprintf(stderr, "Freeing %p\n", (void*)vec->mem); #endif - const DeltaChunk* end = &vec->mem[vec->size]; - DeltaChunk* i; + const DeltaInfo* end = &vec->mem[vec->size]; + DeltaInfo* i; for(i = vec->mem; i < end; i++){ DC_destroy(i); } @@ -306,7 +231,7 @@ void DCV_destroy(DeltaChunkVector* vec) // Reset this vector so that its existing memory can be filled again. // Memory will be kept, but not cleaned up inline -void DCV_forget_members(DeltaChunkVector* vec) +void DIV_forget_members(DeltaInfoVector* vec) { vec->size = 0; } @@ -315,13 +240,13 @@ void DCV_forget_members(DeltaChunkVector* vec) // have been deallocated properly. // It will keep its memory though, and hence can be filled again inline -void DCV_reset(DeltaChunkVector* vec) +void DIV_reset(DeltaInfoVector* vec) { if (vec->size == 0) return; - DeltaChunk* dc = DCV_first(vec); - const DeltaChunk* dcend = DCV_end(vec); + DeltaInfo* dc = DIV_first(vec); + const DeltaInfo* dcend = DIV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); } @@ -333,27 +258,27 @@ void DCV_reset(DeltaChunkVector* vec) // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! static inline -DeltaChunk* DCV_append(DeltaChunkVector* vec) +DeltaInfo* DIV_append(DeltaInfoVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDCV_grow_by); + DIV_grow_by(vec, gDIV_grow_by); } - DeltaChunk* next = vec->mem + vec->size; + DeltaInfo* next = vec->mem + vec->size; vec->size += 1; return next; } // Return delta chunk being closest to the given absolute offset inline -DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) +DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) { assert(vec->mem); ull lo = 0; ull hi = vec->size; ull mid; - DeltaChunk* dc; + DeltaInfo* dc; while (lo < hi) { @@ -368,48 +293,16 @@ DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) } } - return DCV_last(vec); + return DIV_last(vec); } -// Assert the given vector has correct datachunks -// return 1 on success -int DCV_dbg_check_integrity(const DeltaChunkVector* vec) -{ - if(DCV_empty(vec)){ - return 0; - } - const DeltaChunk* i = DCV_first(vec); - const DeltaChunk* end = DCV_end(vec); - - ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); - ull acc_size = 0; - for(; i < end; i++){ - acc_size += i->ts; - } - if (acc_size != aparent_size) - return 0; - - if (vec->size < 2){ - return 1; - } - - const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = DCV_first(vec); i < endm1; i++){ - const DeltaChunk* n = i+1; - if (DC_rbound(i) != n->to){ - return 0; - } - } - - return 1; -} // Return the amount of chunks a slice at the given spot would have inline -uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) +uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) { uint num_dc = 0; - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + DeltaInfo* cdc = DIV_closest_chunk(src, ofs); // partial overlap if (cdc->to != ofs) { @@ -423,7 +316,7 @@ uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) } } - const DeltaChunk* vecend = DCV_end(src); + const DeltaInfo* vecend = DIV_end(src); for( ;(cdc < vecend) && size; ++cdc){ num_dc += 1; if (cdc->ts < size) { @@ -442,12 +335,12 @@ uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) // data chunks // Return: number of chunks in the slice inline -uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, ull size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) { - assert(DCV_lbound(src) <= ofs); - assert((ofs + size) <= DCV_rbound(src)); + assert(DIV_lbound(src) <= ofs); + assert((ofs + size) <= DIV_rbound(src)); - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + DeltaInfo* cdc = DIV_closest_chunk(src, ofs); uint num_chunks = 0; // partial overlap @@ -464,7 +357,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u } } - const DeltaChunk* vecend = DCV_end(src); + const DeltaInfo* vecend = DIV_end(src); for( ;(cdc < vecend) && size; ++cdc) { num_chunks += 1; @@ -483,44 +376,10 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u } -// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which -// originates in to -// 'at' will be replaced by the items to insert ( special purpose ) -// 'at' will be properly destroyed, but all items will just be copied bytewise -// using memcpy. Hence from must just forget about them ! -// IMPORTANT: to must have an appropriate size already -inline -void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) -{ - assert(from->size > 1); - assert(to->size + from->size - 1 <= to->reserved_size); - - // -1 because we replace 'at' - DC_destroy(at); - - // If we are somewhere in the middle, we have to make some space - if (DCV_last(to) != at) { - // IMPORTANT: This memmove kills the performance in case of large deltas - // Causing everything to slow down enormously. Its logical, as the memory - // gets shifted each time we insert nodes, for each chunk, for ever smaller - // chunks depending on the deltas - memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - } - - // Finally copy all the items in - memcpy((void*) at, (void*)DCV_first(from), from->size*sizeof(DeltaChunk)); - - // FINALLY: update size - to->size += from->size - 1; -} - // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. -bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) +bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) { - DBG_check(tdcv); - DBG_check(bdcv); - uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; @@ -529,8 +388,8 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) uint* pofs = offset_array; uint num_addchunks = 0; - DeltaChunk* dc = DCV_first(tdcv); - const DeltaChunk* dcend = DCV_end(tdcv); + DeltaInfo* dc = DIV_first(tdcv); + const DeltaInfo* dcend = DIV_end(tdcv); // OFFSET RUN for (;dc < dcend; dc++, pofs++) @@ -543,14 +402,14 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // offset the next chunk by the amount of chunks in the slice // - 1, because we replace our own chunk - num_addchunks += DCV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + num_addchunks += DIV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; } // reserve enough memory to hold all the new chunks // reinit pointers, array could have been reallocated - DCV_reserve_memory(tdcv, tdcv->size + num_addchunks); - dc = DCV_last(tdcv); - dcend = DCV_first(tdcv) - 1; + DIV_reserve_memory(tdcv, tdcv->size + num_addchunks); + dc = DIV_last(tdcv); + dcend = DIV_first(tdcv) - 1; // now, that we have our pointers with the old size tdcv->size += num_addchunks; @@ -567,7 +426,7 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort if (ofs){ - memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); + memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaInfo)); } continue; } @@ -576,26 +435,24 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // As we could override dc when slicing, we get the data here const ull relofs = dc->to - dc->so; - DeltaChunk* tdc = dc + ofs; - DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); + DeltaInfo* tdc = dc + ofs; + DeltaInfo* tdcend = tdc + DIV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); for(;tdc < tdcend; tdc++){ tdc->to += relofs; } } - DBG_check(tdcv); - PyMem_Free(offset_array); return 1; } // DELTA CHUNK LIST (PYTHON) ///////////////////////////// - +// Internally, it has nothing to do with a ChunkList anymore though typedef struct { PyObject_HEAD // ----------- - DeltaChunkVector vec; + DeltaInfoVector vec; } DeltaChunkList; @@ -608,28 +465,28 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - DCV_init(&self->vec, 0); + DIV_init(&self->vec, 0); return 0; } static void DCL_dealloc(DeltaChunkList* self) { - DCV_destroy(&(self->vec)); + DIV_destroy(&(self->vec)); } static PyObject* DCL_len(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); + return PyLong_FromUnsignedLongLong(DIV_len(&self->vec)); } static inline ull DCL_rbound(DeltaChunkList* self) { - if (DCV_empty(&self->vec)) + if (DIV_empty(&self->vec)) return 0; - return DCV_rbound(&self->vec); + return DIV_rbound(&self->vec); } static @@ -660,7 +517,7 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) } const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DCV_end(&self->vec); + const DeltaChunk* end = DIV_end(&self->vec); const uchar* data; Py_ssize_t dlen; @@ -768,10 +625,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkVector dcv; - DeltaChunkVector tdcv; - DCV_init(&dcv, 100); // should be enough to keep the average text file - DCV_init(&tdcv, 0); + DeltaInfoVector dcv; + DeltaInfoVector tdcv; + DIV_init(&dcv, 100); // should be enough to keep the average text file + DIV_init(&tdcv, 0); unsigned int dsi = 0; PyObject* ds = 0; @@ -796,7 +653,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DCV_reserve_memory(&dcv, approx_num_cmds); + DIV_reserve_memory(&dcv, approx_num_cmds); // parse command stream ull tbw = 0; // Amount of target bytes written @@ -829,7 +686,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) break; } - DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); + DC_init(DIV_append(&dcv), tbw, cp_size, cp_off); tbw += cp_size; } else if (cmd) { @@ -862,7 +719,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #endif - DeltaChunk* dc = DCV_append(&dcv); + DeltaChunk* dc = DIV_append(&dcv); DC_init(dc, tbw, num_bytes, 0); // gather the data, or (possibly) share single blocks @@ -892,7 +749,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } if (!is_first_run){ - if (!DCV_connect_with_base(&tdcv, &dcv)){ + if (!DIV_connect_with_base(&tdcv, &dcv)){ error = 1; } } @@ -905,10 +762,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (is_first_run){ tdcv = dcv; // wipe out dcv without destroying the members, get its own memory - DCV_init(&dcv, tdcv.size); + DIV_init(&dcv, tdcv.size); } else { // destroy members, but keep memory - DCV_reset(&dcv); + DIV_reset(&dcv); } loop_end: @@ -931,7 +788,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (dsi > 1){ // otherwise dcv equals tcl - DCV_destroy(&dcv); + DIV_destroy(&dcv); } // Return the actual python object - its just a container @@ -939,7 +796,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); // Otherwise tdcv would be deallocated by the chunk list - DCV_destroy(&tdcv); + DIV_destroy(&tdcv); error = 1; } else { // Plain copy, don't deallocate From 29fe629dfc11103398f97f0886d79a6f59deeb75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2010 18:37:52 +0200 Subject: [PATCH 0106/3719] Intermediate commit, working my way through the code, step by step. Didn't even try to compile it yet --- _delta_apply.c | 183 +++++++++++++++++++++++++++++++++++++------------ stream.py | 13 +--- 2 files changed, 144 insertions(+), 52 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 9768b5264..bd09bc5ed 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -14,6 +14,24 @@ typedef uchar bool; const ull gDIV_grow_by = 100; +// DELTA STREAM ACCESS +/////////////////////// +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; + return size; +} + + // DELTA INFO ///////////// typedef struct { @@ -22,6 +40,65 @@ typedef struct { } DeltaInfo; +// TOP LEVEL STREAM INFO +///////////////////////////// +typedef struct { + const uchar* tds; + Py_ssize_t* tdslen; + Py_ssize_t target_size; // size of the target buffer which can hold all data + PyObject* parent_object; +} ToplevelStreamInfo; + + +void TSI_init(ToplevelStreamInfo* info) +{ + info->tds = 0; + info->tdslen = 0; + info->target_size = 0; + info->parent_object = 0; + +} + +void TSI_destroy(ToplevelStreamInfo* info) +{ + if (info->parent_object){ + Py_DECREF(info->parent_object); + info->parent_object = 0; + } else if (info->tds){ + PyMem_Free(info->tds); + } +} + +// initialize our set stream to point to the first chunk +// Fill in the header information, which is the base and target size +void TSI_init_stream(ToplevelStreamInfo* info) +{ + assert(info->tds && info->tdslen) + + // init stream + const uchar* tdsend = info->tds + info->tdslen; + msb_size(&info->tds, tdsend); + info->target_size = msb_size(&info->tds, tdsend); +} + +// duplicate the data currently owned by the parent object drop its refcount +// return 1 on success +bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) +{ + assert(info.parent_object); + + uchar* ptmp = PyMem_Malloc(info.tdslen); + if (!ptmp){ + return 0; + } + memcpy((void*)ptmp, info.tds, info.tdslen); + tds = ptmp; + Py_DECREF(info.parent_object); + info.parent_object = 0; + + return 1; +} + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -452,7 +529,7 @@ bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) typedef struct { PyObject_HEAD // ----------- - DeltaInfoVector vec; + ToplevelStreamInfo istream; } DeltaChunkList; @@ -465,34 +542,20 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - DIV_init(&self->vec, 0); + TSI_init(&self->istream, 0); return 0; } static void DCL_dealloc(DeltaChunkList* self) { - DIV_destroy(&(self->vec)); -} - -static -PyObject* DCL_len(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DIV_len(&self->vec)); -} - -static inline -ull DCL_rbound(DeltaChunkList* self) -{ - if (DIV_empty(&self->vec)) - return 0; - return DIV_rbound(&self->vec); + TSI_destroy(&(self->istream)); } static PyObject* DCL_py_rbound(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(DCL_rbound(self)); + return PyLong_FromUnsignedLongLong(self->istream->target_size); } // Write using a write function, taking remaining bytes from a base buffer @@ -535,7 +598,6 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -596,21 +658,6 @@ DeltaChunkList* DCL_new_instance(void) return dcl; } -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator @@ -626,22 +673,71 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } DeltaInfoVector dcv; - DeltaInfoVector tdcv; + ToplevelStreamInfo tdsinfo; + TSI_init(&tdsinfo); DIV_init(&dcv, 100); // should be enough to keep the average text file - DIV_init(&tdcv, 0); - unsigned int dsi = 0; - PyObject* ds = 0; + + // GET TOPLEVEL DELTA STREAM int error = 0; - for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + PyObject* ds = 0; + unsigned int dsi = 0; + ds = PyIter_Next(stream_iter); + if (!ds){ + error = 1; + goto _error; + } + + dsi += 1; + tdsinfo.parent_object = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(tdsinfo.parent_object)){ + Py_DECREF(ds); + error = 1; + goto _error; + } + + PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); + if (tdslen > pow(2, 32)){ + // parent object is deallocated by info structure + Py_DECREF(ds); + PyErr_SetString(PyExc_RuntimeError("Cannot handle deltas larger than 4GB")); + tdsinfo.tdb = 0; + + error = 1; + goto _error; + } + Py_DECREF(ds); + + // INTEGRATE ANCESTOR DELTA STREAMS + PyObject* db = 0; + TSI_init_stream(&tdsinfo, tdb); + + + for (ds = PyIter_Next(stream_iter); ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) { - PyObject* db = PyObject_CallMethod(ds, "read", 0); + // Its important to initialize this before the next block which can jump + // to code who needs this to exist ! + PyObject* db = 0; + + // When processing the first delta, we know we will have to alter the tds + // Hence we copy it and deallocate the parent object + if (ds == 1) { + if (!TSI_copy_stream_from_object(&tdsinfo)){ + PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); + // info structure takes care of the parent_object + error = 1; + goto loop_end; + } + } + + db = PyObject_CallMethod(ds, "read", 0); if (!PyObject_CheckReadBuffer(db)){ error = 1; PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); goto loop_end; } + // Fill the stream info structure const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); @@ -778,10 +874,13 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } }// END for each stream object - if (dsi == 0 && ! error){ + if (dsi == 0){ PyErr_SetString(PyExc_ValueError, "No streams provided"); } + +_error: + if (stream_iter != dstreams){ Py_DECREF(stream_iter); } @@ -800,7 +899,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; } else { // Plain copy, don't deallocate - dcl->vec = tdcv; + dcl->istream = tdsinfo; } if (error){ diff --git a/stream.py b/stream.py index 38c86dae7..0d8972898 100644 --- a/stream.py +++ b/stream.py @@ -349,7 +349,7 @@ def _set_cache_too_slow_without_c(self, attr): # call len directly, as the (optional) c version doesn't implement the sequence # protocol - if dcl.__len__() == 0: + if dcl.rbound() == 0: self._size = 0 self._mm_target = allocate_memory(0) return @@ -367,15 +367,6 @@ def _set_cache_too_slow_without_c(self, attr): self._mm_target.seek(0) - def _set_cache_(self, attr): - """Determine which version to use depending on the configuration of the deltas - :note: we are only called if we have the performance module""" - # otherwise it depends on the amount of memory to shift around - if len(self._dstreams) > 1 and self._bstream.size < 150000: - return self._set_cache_too_slow_without_c(attr) - else: - return self._set_cache_brute_(attr) - def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" @@ -456,6 +447,8 @@ def _set_cache_brute_(self, attr): #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c #} END configuration From 2414fd48969bf5bff510c8ded83edd7b4900626b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Oct 2010 21:06:17 +0200 Subject: [PATCH 0107/3719] Brutally made code compile, most of the major functions are still commented out, but it should just be a matter of time until its back and working --- _delta_apply.c | 324 ++++++++++++++++++++++--------------------------- 1 file changed, 142 insertions(+), 182 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index bd09bc5ed..abeaed5f1 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -32,31 +32,24 @@ ull msb_size(const uchar** datap, const uchar* top) } -// DELTA INFO -///////////// -typedef struct { - uint dso; // delta stream offset - uint to; // target offset (cache) -} DeltaInfo; - - // TOP LEVEL STREAM INFO ///////////////////////////// typedef struct { - const uchar* tds; - Py_ssize_t* tdslen; - Py_ssize_t target_size; // size of the target buffer which can hold all data - PyObject* parent_object; + const uchar *tds; + Py_ssize_t tdslen; // size of tds in bytes + Py_ssize_t target_size; // size of the target buffer which can hold all data + uint numChunks; // amount of chunks in the delta stream + PyObject *parent_object; } ToplevelStreamInfo; void TSI_init(ToplevelStreamInfo* info) { - info->tds = 0; + info->tds = NULL; info->tdslen = 0; + info->numChunks = 0; info->target_size = 0; info->parent_object = 0; - } void TSI_destroy(ToplevelStreamInfo* info) @@ -65,7 +58,7 @@ void TSI_destroy(ToplevelStreamInfo* info) Py_DECREF(info->parent_object); info->parent_object = 0; } else if (info->tds){ - PyMem_Free(info->tds); + PyMem_Free((void*)info->tds); } } @@ -73,7 +66,7 @@ void TSI_destroy(ToplevelStreamInfo* info) // Fill in the header information, which is the base and target size void TSI_init_stream(ToplevelStreamInfo* info) { - assert(info->tds && info->tdslen) + assert(info->tds && info->tdslen); // init stream const uchar* tdsend = info->tds + info->tdslen; @@ -85,16 +78,16 @@ void TSI_init_stream(ToplevelStreamInfo* info) // return 1 on success bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) { - assert(info.parent_object); + assert(info->parent_object); - uchar* ptmp = PyMem_Malloc(info.tdslen); + uchar* ptmp = PyMem_Malloc(info->tdslen); if (!ptmp){ return 0; } - memcpy((void*)ptmp, info.tds, info.tdslen); - tds = ptmp; - Py_DECREF(info.parent_object); - info.parent_object = 0; + memcpy((void*)ptmp, info->tds, info->tdslen); + info->tds = ptmp; + Py_DECREF(info->parent_object); + info->parent_object = 0; return 1; } @@ -152,11 +145,21 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec // DELTA CHUNK VECTOR ///////////////////// + +// DELTA INFO +///////////// +typedef struct { + uint dso; // delta stream offset + uint to; // target offset (cache) +} DeltaInfo; + + typedef struct { - DeltaInfo* mem; // Memory - const uchar* dstream; // pointer to delta stream we index - Py_ssize_t size; // Size in DeltaInfos - Py_ssize_t reserved_size; // Reserve in DeltaInfos + DeltaInfo *mem; // Memory + uint di_last_size; // size of the last element - we can't compute it using the next bound + const uchar *dstream; // pointer to delta stream we index - its borrowed + Py_ssize_t size; // Amount of DeltaInfos + Py_ssize_t reserved_size; // Reserved amount of DeltaInfos } DeltaInfoVector; @@ -164,7 +167,7 @@ typedef struct { // Reserve enough memory to hold the given amount of delta chunks // Return 1 on success // NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DCVs change often, and reallocs are expensive +// just for a single additional entry. DIVs change often, and reallocs are expensive inline int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) { @@ -219,18 +222,19 @@ int DIV_init(DeltaInfoVector* vec, ull initial_size) vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; + vec->di_last_size = 0; return DIV_grow_by(vec, initial_size); } inline -ull DIV_len(const DeltaInfoVector* vec) +Py_ssize_t DIV_len(const DeltaInfoVector* vec) { return vec->size; } inline -ull DIV_lbound(const DeltaInfoVector* vec) +uint DIV_lbound(const DeltaInfoVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -251,18 +255,6 @@ DeltaInfo* DIV_last(const DeltaInfoVector* vec) return DIV_get(vec, vec->size-1); } -inline -ull DIV_rbound(const DeltaInfoVector* vec) -{ - return DC_rbound(DIV_last(vec)); -} - -inline -ull DIV_size(const DeltaInfoVector* vec) -{ - return DIV_rbound(vec) - DIV_lbound(vec); -} - inline int DIV_empty(const DeltaInfoVector* vec) { @@ -285,19 +277,24 @@ DeltaInfo* DIV_first(const DeltaInfoVector* vec) return vec->mem; } +// return rbound offset in bytes. We use information contained in the +// vec to do that +inline +uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) +{ + if (DIV_last(vec) == di){ + return di->to + vec->di_last_size; + } else { + return (di+1)->to; + } +} + void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG fprintf(stderr, "Freeing %p\n", (void*)vec->mem); #endif - - const DeltaInfo* end = &vec->mem[vec->size]; - DeltaInfo* i; - for(i = vec->mem; i < end; i++){ - DC_destroy(i); - } - PyMem_Free(vec->mem); vec->size = 0; vec->reserved_size = 0; @@ -313,21 +310,13 @@ void DIV_forget_members(DeltaInfoVector* vec) vec->size = 0; } -// Reset the vector so that its size will be zero, and its members will -// have been deallocated properly. +// Reset the vector so that its size will be zero // It will keep its memory though, and hence can be filled again inline void DIV_reset(DeltaInfoVector* vec) { if (vec->size == 0) return; - - DeltaInfo* dc = DIV_first(vec); - const DeltaInfo* dcend = DIV_end(vec); - for(;dc < dcend; dc++){ - DC_destroy(dc); - } - vec->size = 0; } @@ -363,7 +352,7 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) dc = vec->mem + mid; if (dc->to > ofs){ hi = mid; - } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + } else if ((DIV_info_rbound(vec, dc) > ofs) | (dc->to == ofs)) { return dc; } else { lo = mid + 1; @@ -378,7 +367,7 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) inline uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) { - uint num_dc = 0; + /*uint num_dc = 0; DeltaInfo* cdc = DIV_closest_chunk(src, ofs); // partial overlap @@ -404,7 +393,9 @@ uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) } } - return num_dc; + return num_dc;*/ + assert(0); // TODO + return 0; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -414,8 +405,9 @@ uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) { + /* assert(DIV_lbound(src) <= ofs); - assert((ofs + size) <= DIV_rbound(src)); + assert((ofs + size) <= DIV_last(src)->to + src->di_last_size); DeltaInfo* cdc = DIV_closest_chunk(src, ofs); uint num_chunks = 0; @@ -450,13 +442,17 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull assert(size == 0); return num_chunks; + */ + assert(0); // TODO + return 0; } -// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. -bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) +// Take slices of div into the corresponding area of the tsi, which is the topmost +// delta to apply. +bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { + /* uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; @@ -521,6 +517,9 @@ bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) PyMem_Free(offset_array); return 1; + */ + assert(0); // TODO + return 0; } // DELTA CHUNK LIST (PYTHON) @@ -542,7 +541,7 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - TSI_init(&self->istream, 0); + TSI_init(&self->istream); return 0; } @@ -555,13 +554,14 @@ void DCL_dealloc(DeltaChunkList* self) static PyObject* DCL_py_rbound(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(self->istream->target_size); + return PyLong_FromUnsignedLongLong(self->istream.target_size); } // Write using a write function, taking remaining bytes from a base buffer static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + /* PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -593,6 +593,9 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) } Py_DECREF(tmpargs); + */ + // TODO + assert(0); Py_RETURN_NONE; } @@ -653,11 +656,51 @@ DeltaChunkList* DCL_new_instance(void) assert(dcl); DCL_init(dcl, 0, 0); - assert(dcl->vec.size == 0); - assert(dcl->vec.mem == NULL); return dcl; } +// Read the next delta chunk from the given stream and advance it +// dc will contain the parsed information, its offset must be set by +// the previous call of next_delta_info, which implies it should remain the +// same instance between the calls. +// Return 1 on success, 0 on failure +inline +bool next_delta_info(const uchar** dstream, DeltaChunk* dc) +{ + const uchar* data = *dstream; + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + dc->to += dc->ts; + dc->data = 0; + dc->so = cp_off; + dc->ts = cp_size; + + } else if (cmd) { + // Just share the data + dc->to += dc->ts; + dc->data = data; + dc->ts = cmd; + dc->so = 0; + } else { + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + return 0; + } + + return 1; +} + static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator @@ -672,16 +715,16 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaInfoVector dcv; + DeltaInfoVector div; ToplevelStreamInfo tdsinfo; TSI_init(&tdsinfo); - DIV_init(&dcv, 100); // should be enough to keep the average text file + DIV_init(&div, 100); // should be enough to keep the average text file // GET TOPLEVEL DELTA STREAM int error = 0; PyObject* ds = 0; - unsigned int dsi = 0; + unsigned int dsi = 0; // delta stream index we process ds = PyIter_Next(stream_iter); if (!ds){ error = 1; @@ -697,11 +740,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); - if (tdslen > pow(2, 32)){ + if (tdsinfo.tdslen > pow(2, 32)){ // parent object is deallocated by info structure Py_DECREF(ds); - PyErr_SetString(PyExc_RuntimeError("Cannot handle deltas larger than 4GB")); - tdsinfo.tdb = 0; + PyErr_SetString(PyExc_RuntimeError, "Cannot handle deltas larger than 4GB"); + tdsinfo.parent_object = 0; error = 1; goto _error; @@ -709,11 +752,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(ds); // INTEGRATE ANCESTOR DELTA STREAMS - PyObject* db = 0; - TSI_init_stream(&tdsinfo, tdb); + TSI_init_stream(&tdsinfo); - for (ds = PyIter_Next(stream_iter); ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) { // Its important to initialize this before the next block which can jump // to code who needs this to exist ! @@ -721,7 +763,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // When processing the first delta, we know we will have to alter the tds // Hence we copy it and deallocate the parent object - if (ds == 1) { + if (dsi == 1) { if (!TSI_copy_stream_from_object(&tdsinfo)){ PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); // info structure takes care of the parent_object @@ -744,125 +786,45 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* dend = data + dlen; // read header - const ull base_size = msb_size(&data, dend); + msb_size(&data, dend); const ull target_size = msb_size(&data, dend); // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DIV_reserve_memory(&dcv, approx_num_cmds); + DIV_reserve_memory(&div, approx_num_cmds); // parse command stream - ull tbw = 0; // Amount of target bytes written - bool is_shared_data = dsi != 0; - bool is_first_run = dsi == 0; + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); while (data < dend) { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - const unsigned long rbound = cp_off + cp_size; - if (rbound < cp_size || - rbound > base_size){ - // this really shouldn't happen - error = 1; - assert(0); - break; - } - - DC_init(DIV_append(&dcv), tbw, cp_size, cp_off); - tbw += cp_size; - - } else if (cmd) { - // Compression reduces fragmentation though, which is why we do it - // in all cases. - // It makes the more sense the more consecutive add-chunks we have, - // its more likely in big deltas, for big binary files - const uchar* add_start = data - 1; - const uchar* add_end = dend; - ull num_bytes = cmd; - data += cmd; - ull num_chunks = 1; - while (data < dend){ - //while (0){ - const char c = *data; - if (c & 0x80){ - add_end = data; - break; - } else { - data += 1 + c; // advance by 1 to skip add cmd - num_bytes += c; - num_chunks += 1; - } - } - - #ifdef DEBUG - assert(add_end - add_start > 0); - if (num_chunks > 1){ - fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); - } - #endif - - DeltaChunk* dc = DIV_append(&dcv); - DC_init(dc, tbw, num_bytes, 0); - - // gather the data, or (possibly) share single blocks - if (num_chunks > 1){ - uchar* dcdata = PyMem_Malloc(num_bytes); - while (add_start < add_end){ - const char bytes = *add_start++; - memcpy((void*)dcdata, (void*)add_start, bytes); - dcdata += bytes; - add_start += bytes; - } - DC_set_data_with_ownership(dc, dcdata-num_bytes); - } else { - DC_set_data(dc, data - cmd, cmd, is_shared_data); - } - - tbw += num_bytes; - } else { + if (next_delta_info(&data, &dc)){ + // TODO + assert(0); + } else { error = 1; - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); goto loop_end; } }// END handle command opcodes - if (tbw != target_size){ + + if (DC_rbound(&dc) != target_size){ PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); error = 1; } - if (!is_first_run){ - if (!DIV_connect_with_base(&tdcv, &dcv)){ - error = 1; - } + if (!DIV_connect_with_base(&tdsinfo, &div)){ + error = 1; } - + #ifdef DEBUG - fprintf(stderr, "tdcv->size = %i, tdcv->reserved_size = %i\n", (int)tdcv.size, (int)tdcv.reserved_size); - fprintf(stderr, "dcv->size = %i, dcv->reserved_size = %i\n", (int)dcv.size, (int)dcv.reserved_size); + fprintf(stderr, "tdsinfo->len = %i\n", (int)tdsinfo.tdslen); + fprintf(stderr, "div->size = %i, div->reserved_size = %i\n", (int)div.size, (int)div.reserved_size); #endif - if (is_first_run){ - tdcv = dcv; - // wipe out dcv without destroying the members, get its own memory - DIV_init(&dcv, tdcv.size); - } else { - // destroy members, but keep memory - DIV_reset(&dcv); - } + // destroy members, but keep memory + DIV_reset(&div); loop_end: // perform cleanup @@ -885,20 +847,18 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - if (dsi > 1){ - // otherwise dcv equals tcl - DIV_destroy(&dcv); - } + + DIV_destroy(&div); // Return the actual python object - its just a container DeltaChunkList* dcl = DCL_new_instance(); if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdcv would be deallocated by the chunk list - DIV_destroy(&tdcv); + // Otherwise tdsinfo would be deallocated by the chunk list + TSI_destroy(&tdsinfo); error = 1; } else { - // Plain copy, don't deallocate + // Plain copy, transfer ownership to dcl dcl->istream = tdsinfo; } From 9e62c5481fefcc9e8adf0b6387952e214295223c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 00:52:43 +0200 Subject: [PATCH 0108/3719] Worked my way up to re-encoding delta chunks, connect_with method still needs some work, intermediate commit --- _delta_apply.c | 309 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 243 insertions(+), 66 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index abeaed5f1..d35cb7ebc 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -8,6 +8,7 @@ typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; +typedef unsigned short ushort; typedef uchar bool; // Constants @@ -35,10 +36,11 @@ ull msb_size(const uchar** datap, const uchar* top) // TOP LEVEL STREAM INFO ///////////////////////////// typedef struct { - const uchar *tds; + const uchar *tds; // Toplevel delta stream + const uchar *cstart; // start of the chunks Py_ssize_t tdslen; // size of tds in bytes Py_ssize_t target_size; // size of the target buffer which can hold all data - uint numChunks; // amount of chunks in the delta stream + uint num_chunks; // amount of chunks in the delta stream PyObject *parent_object; } ToplevelStreamInfo; @@ -46,8 +48,9 @@ typedef struct { void TSI_init(ToplevelStreamInfo* info) { info->tds = NULL; + info->cstart = NULL; info->tdslen = 0; - info->numChunks = 0; + info->num_chunks = 0; info->target_size = 0; info->parent_object = 0; } @@ -62,18 +65,37 @@ void TSI_destroy(ToplevelStreamInfo* info) } } +inline +const uchar* TSI_end(ToplevelStreamInfo* info) +{ + return info->tds + info->tdslen; +} + +inline +const uchar* TSI_first(ToplevelStreamInfo* info) +{ + return info->cstart; +} + +// set the stream, and initialize it // initialize our set stream to point to the first chunk // Fill in the header information, which is the base and target size -void TSI_init_stream(ToplevelStreamInfo* info) +inline +void TSI_set_stream(ToplevelStreamInfo* info, const uchar* stream) { + info->tds = stream; + info->cstart = stream; + assert(info->tds && info->tdslen); // init stream - const uchar* tdsend = info->tds + info->tdslen; - msb_size(&info->tds, tdsend); - info->target_size = msb_size(&info->tds, tdsend); + const uchar* tdsend = TSI_end(info); + msb_size(&info->cstart, tdsend); // base size + info->target_size = msb_size(&info->cstart, tdsend); } + + // duplicate the data currently owned by the parent object drop its refcount // return 1 on success bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) @@ -86,12 +108,29 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) } memcpy((void*)ptmp, info->tds, info->tdslen); info->tds = ptmp; + info->cstart = ptmp; Py_DECREF(info->parent_object); info->parent_object = 0; return 1; } +// make sure we have the given amount of memory available. This will change +// our official length in bytes right away, its up to the caller +// to do something useful with the freed space +// Return true on success +bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) +{ + assert(info->tds); + if (num_bytes <= info->tdslen){ + return 1; + } + info->tds = PyMem_Realloc((void*)info->tds, num_bytes); + info->cstart = info->tds; + + return info->tds != NULL; +} + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -99,8 +138,8 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) // The data pointer is always shared typedef struct { ull to; - ull ts; - ull so; + uint ts; + uint so; const uchar* data; } DeltaChunk; @@ -141,9 +180,66 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec } +// Encode the information in the given delta chunk and write the byte-stream +// into the given output stream +inline +void DC_encode_to(const DeltaChunk* dc, uchar** pout) +{ + uchar* out = *pout; + if (dc->data){ + *out++ = (uchar)dc->ts; + memcpy(out, dc->data, dc->ts); + out += dc->ts; + } else { + uchar i = 0x80; + uchar* op = out++; + uint moff = dc->so; + uint msize = dc->ts; + + if (moff & 0x000000ff) + *out++ = moff >> 0, i |= 0x01; + if (moff & 0x0000ff00) + *out++ = moff >> 8, i |= 0x02; + if (moff & 0x00ff0000) + *out++ = moff >> 16, i |= 0x04; + if (moff & 0xff000000) + *out++ = moff >> 24, i |= 0x08; + + if (msize & 0x00ff) + *out++ = msize >> 0, i |= 0x10; + if (msize & 0xff00) + *out++ = msize >> 8, i |= 0x20; + + *op = i; + } + *pout = out; +} + +// Return: amount of bytes one would need to encode dc +inline +ushort DC_count_encode_bytes(const DeltaChunk* dc) +{ + if (dc->data){ + return 1 + dc->ts; // cmd byte + actual data bytes + } else { + ushort c = 1; // cmd byte + uint ts = dc->ts; + ull to = dc->to; + + // offset + c += to & 0x000000FF; + c += to & 0x0000FF00; + c += to & 0x00FF0000; + c += to & 0xFF000000; + + // size - max size is 0x10000, its encoded with 0 size bits + c += ts & 0x000000FF; + c += ts & 0x0000FF00; + + return c; + } +} -// DELTA CHUNK VECTOR -///////////////////// // DELTA INFO @@ -154,10 +250,13 @@ typedef struct { } DeltaInfo; +// DELTA INFO VECTOR +////////////////////// + typedef struct { - DeltaInfo *mem; // Memory - uint di_last_size; // size of the last element - we can't compute it using the next bound - const uchar *dstream; // pointer to delta stream we index - its borrowed + DeltaInfo *mem; // Memory for delta infos + uint di_last_size; // size of the last element - we can't compute it using the next bound + const uchar *dstream; // borrowed ointer to delta stream we index Py_ssize_t size; // Amount of DeltaInfos Py_ssize_t reserved_size; // Reserved amount of DeltaInfos } DeltaInfoVector; @@ -220,6 +319,7 @@ int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) int DIV_init(DeltaInfoVector* vec, ull initial_size) { vec->mem = NULL; + vec->dstream = NULL; vec->size = 0; vec->reserved_size = 0; vec->di_last_size = 0; @@ -289,6 +389,24 @@ uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) } } +// return size of the given delta info item +inline +uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo const* veclast) +{ + if (veclast == di){ + return vec->di_last_size; + } else { + return (di+1)->to - di->to; + } +} + +// return size of the given delta info item +inline +uint DIV_info_size(const DeltaInfoVector* vec, const DeltaInfo* di) +{ + return DIV_info_size2(vec, di, DIV_last(vec)); +} + void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ @@ -344,16 +462,16 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) ull lo = 0; ull hi = vec->size; ull mid; - DeltaInfo* dc; + DeltaInfo* di; while (lo < hi) { mid = (lo + hi) / 2; - dc = vec->mem + mid; - if (dc->to > ofs){ + di = vec->mem + mid; + if (di->to > ofs){ hi = mid; - } else if ((DIV_info_rbound(vec, dc) > ofs) | (dc->to == ofs)) { - return dc; + } else if ((DIV_info_rbound(vec, di) > ofs) | (di->to == ofs)) { + return di; } else { lo = mid + 1; } @@ -362,40 +480,58 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) return DIV_last(vec); } +// forward declaration +const uchar* next_delta_info(const uchar*, DeltaChunk*); -// Return the amount of chunks a slice at the given spot would have +// Return the amount of chunks a slice at the given spot would have, as well as +// its size in bytes it would have if the possibly partial chunks would be encoded +// The bytes will be added inline -uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) +uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull size, uint* out_bytes) { - /*uint num_dc = 0; - DeltaInfo* cdc = DIV_closest_chunk(src, ofs); + uint num_dc = 0; + DeltaInfo* cdi = DIV_closest_chunk(src, ofs); + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); // partial overlap - if (cdc->to != ofs) { - const ull relofs = ofs - cdc->to; - size -= cdc->ts - relofs < size ? cdc->ts - relofs : size; + if (cdi->to != ofs) { + const ull relofs = ofs - cdi->to; + const uint cdisize = DIV_info_size(src, cdi); + size -= cdisize - relofs < size ? cdisize - relofs : size; num_dc += 1; - cdc += 1; + cdi += 1; + + // get the size in bytes the info would have + next_delta_info(src->dstream + cdi->dso, &dc); + *out_bytes += DC_count_encode_bytes(&dc); if (size == 0){ return num_dc; } } - const DeltaInfo* vecend = DIV_end(src); - for( ;(cdc < vecend) && size; ++cdc){ + const DeltaInfo const* vecend = DIV_end(src); + const DeltaInfo const* veclast = DIV_last(src); + for( ;(cdi < vecend) && size; ++cdi){ num_dc += 1; - if (cdc->ts < size) { - size -= cdc->ts; + + const uint cdisize = DIV_info_size2(src, cdi, veclast); + + next_delta_info(src->dstream + cdi->dso, &dc); + *out_bytes += DC_count_encode_bytes(&dc); + + if (cdisize < size) { + size -= cdisize; } else { size = 0; break; } } - return num_dc;*/ - assert(0); // TODO - return 0; + *out_bytes += 0; + return num_dc; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -447,40 +583,47 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull return 0; } - // Take slices of div into the corresponding area of the tsi, which is the topmost // delta to apply. bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { - /* - uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + assert(tsi->num_chunks); + + uint *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(uint)); if (!offset_array){ return 0; } uint* pofs = offset_array; uint num_addchunks = 0; + uint num_addbytes = 0; - DeltaInfo* dc = DIV_first(tdcv); - const DeltaInfo* dcend = DIV_end(tdcv); + const uchar* data = TSI_first(tsi); + const uchar const* dend = TSI_end(tsi); + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); // OFFSET RUN - for (;dc < dcend; dc++, pofs++) + for (;data < dend; pofs++) { // Data chunks don't need processing *pofs = num_addchunks; - if (dc->data){ + data = next_delta_info(data, &dc); + + if (dc.data){ continue; } // offset the next chunk by the amount of chunks in the slice // - 1, because we replace our own chunk - num_addchunks += DIV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + num_addchunks += DIV_count_slice_chunks_and_bytes(div, dc.so, dc.ts, &num_addbytes) - 1; + assert(num_addbytes); } + /* // reserve enough memory to hold all the new chunks // reinit pointers, array could have been reallocated - DIV_reserve_memory(tdcv, tdcv->size + num_addchunks); + TSI_resize(tsis, tsi->tdslen + num_addbytes); dc = DIV_last(tdcv); dcend = DIV_first(tdcv) - 1; @@ -514,12 +657,10 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) tdc->to += relofs; } } - + */ PyMem_Free(offset_array); return 1; - */ - assert(0); // TODO - return 0; + } // DELTA CHUNK LIST (PYTHON) @@ -663,23 +804,22 @@ DeltaChunkList* DCL_new_instance(void) // dc will contain the parsed information, its offset must be set by // the previous call of next_delta_info, which implies it should remain the // same instance between the calls. -// Return 1 on success, 0 on failure +// Return the altered uchar pointer, reassign it to the input data inline -bool next_delta_info(const uchar** dstream, DeltaChunk* dc) +const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) { - const uchar* data = *dstream; const char cmd = *data++; if (cmd & 0x80) { - unsigned long cp_off = 0, cp_size = 0; + uint cp_off = 0, cp_size = 0; if (cmd & 0x01) cp_off = *data++; if (cmd & 0x02) cp_off |= (*data++ << 8); if (cmd & 0x04) cp_off |= (*data++ << 16); if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); if (cmd & 0x10) cp_size = *data++; if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cmd & 0x40) cp_size |= (*data++ << 16); // this should never get hit with current deltas ... if (cp_size == 0) cp_size = 0x10000; dc->to += dc->ts; @@ -695,10 +835,34 @@ bool next_delta_info(const uchar** dstream, DeltaChunk* dc) dc->so = 0; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - return 0; + return NULL; } - return 1; + return data; +} + +// Return amount of chunks encoded in the given delta stream +// If read_header is True, then the header msb chunks will be read first. +// Otherwise, the stream is assumed to be scrubbed one past the header +uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) +{ + // read header + if (read_header){ + msb_size(&data, dend); + msb_size(&data, dend); + } + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + uint num_chunks = 0; + + while (data < dend) + { + data = next_delta_info(data, &dc); + num_chunks += 1; + }// END handle command opcodes + + return num_chunks; } static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) @@ -751,10 +915,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } Py_DECREF(ds); - // INTEGRATE ANCESTOR DELTA STREAMS - TSI_init_stream(&tdsinfo); - + // let it officially know, and initialize its internal state + TSI_set_stream(&tdsinfo, tdsinfo.tds); + // INTEGRATE ANCESTOR DELTA STREAMS for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) { // Its important to initialize this before the next block which can jump @@ -770,6 +934,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; goto loop_end; } + + tdsinfo.num_chunks = compute_chunk_count(tdsinfo.cstart, TSI_end(&tdsinfo), 0); } db = PyObject_CallMethod(ds, "read", 0); @@ -783,37 +949,48 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* dend = data + dlen; + const uchar const* dstart = data; + const uchar const* dend = data + dlen; + div.dstream = dstart; + + if (dlen > pow(2, 32)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Cannot currently handle deltas larger than 4GB"); + goto loop_end; + } - // read header + // READ HEADER msb_size(&data, dend); const ull target_size = msb_size(&data, dend); - // Assume good compression for the adds - const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DIV_reserve_memory(&div, approx_num_cmds); + DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); // parse command stream DeltaChunk dc; + DeltaInfo* di = 0; // temporary pointer DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); while (data < dend) { - if (next_delta_info(&data, &dc)){ - // TODO - assert(0); + di = DIV_append(&div); + di->dso = data - dstart; + if ((data = next_delta_info(data, &dc))){ + di->to = dc.to; } else { error = 1; goto loop_end; } }// END handle command opcodes + // finalize information + div.di_last_size = dc.ts; + if (DC_rbound(&dc) != target_size){ PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); error = 1; } - + if (!DIV_connect_with_base(&tdsinfo, &div)){ error = 1; } From 8693a7e37af03cd5d36f36fcc5ed01e938798905 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 12:26:07 +0200 Subject: [PATCH 0109/3719] Implemented connect_with which includes all the slicing functions, which now operate on the delta stream data directly, its yet to be tested though, and I am afraid of this --- _delta_apply.c | 207 ++++++++++++++++++++++++++++--------------------- 1 file changed, 118 insertions(+), 89 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index d35cb7ebc..5f1d9c379 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -106,9 +106,12 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) if (!ptmp){ return 0; } + uint ofs = (uint)(info->cstart - info->tds); memcpy((void*)ptmp, info->tds, info->tdslen); + info->tds = ptmp; - info->cstart = ptmp; + info->cstart = ptmp + ofs; + Py_DECREF(info->parent_object); info->parent_object = 0; @@ -125,8 +128,10 @@ bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) if (num_bytes <= info->tdslen){ return 1; } + uint ofs = (uint)(info->cstart - info->tds); info->tds = PyMem_Realloc((void*)info->tds, num_bytes); - info->cstart = info->tds; + info->tdslen = num_bytes; + info->cstart = info->tds + ofs; return info->tds != NULL; } @@ -182,19 +187,21 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec // Encode the information in the given delta chunk and write the byte-stream // into the given output stream +// It will be copied into the given bounds, the given size must be the final size +// and work with the given relative offset - hence the bounds are assumed to be +// correct and to fit within the unaltered dc inline -void DC_encode_to(const DeltaChunk* dc, uchar** pout) +void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; if (dc->data){ - *out++ = (uchar)dc->ts; - memcpy(out, dc->data, dc->ts); - out += dc->ts; + *out++ = (uchar)size; + memcpy(out, dc->data+ofs, size); + out += size; } else { uchar i = 0x80; uchar* op = out++; - uint moff = dc->so; - uint msize = dc->ts; + uint moff = dc->so+ofs; if (moff & 0x000000ff) *out++ = moff >> 0, i |= 0x01; @@ -205,10 +212,10 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout) if (moff & 0xff000000) *out++ = moff >> 24, i |= 0x08; - if (msize & 0x00ff) - *out++ = msize >> 0, i |= 0x10; - if (msize & 0xff00) - *out++ = msize >> 8, i |= 0x20; + if (size & 0x00ff) + *out++ = size >> 0, i |= 0x10; + if (size & 0xff00) + *out++ = size >> 8, i |= 0x20; *op = i; } @@ -224,13 +231,13 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) } else { ushort c = 1; // cmd byte uint ts = dc->ts; - ull to = dc->to; + ull so = dc->so; // offset - c += to & 0x000000FF; - c += to & 0x0000FF00; - c += to & 0x00FF0000; - c += to & 0xFF000000; + c += so & 0x000000FF; + c += so & 0x0000FF00; + c += so & 0x00FF0000; + c += so & 0xFF000000; // size - max size is 0x10000, its encoded with 0 size bits c += ts & 0x000000FF; @@ -485,13 +492,15 @@ const uchar* next_delta_info(const uchar*, DeltaChunk*); // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded -// The bytes will be added +// and added to the spot marked by sdc inline -uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull size, uint* out_bytes) +uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) { - uint num_dc = 0; + uint num_bytes = 0; DeltaInfo* cdi = DIV_closest_chunk(src, ofs); + + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -499,63 +508,72 @@ uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull s if (cdi->to != ofs) { const ull relofs = ofs - cdi->to; const uint cdisize = DIV_info_size(src, cdi); - size -= cdisize - relofs < size ? cdisize - relofs : size; - num_dc += 1; - cdi += 1; + const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + size -= actual_size; // get the size in bytes the info would have next_delta_info(src->dstream + cdi->dso, &dc); - *out_bytes += DC_count_encode_bytes(&dc); + dc.so += relofs; + dc.ts = actual_size; + num_bytes += DC_count_encode_bytes(&dc); + + cdi += 1; if (size == 0){ - return num_dc; + return num_bytes; } } const DeltaInfo const* vecend = DIV_end(src); - const DeltaInfo const* veclast = DIV_last(src); - for( ;(cdi < vecend) && size; ++cdi){ - num_dc += 1; - - const uint cdisize = DIV_info_size2(src, cdi, veclast); - + for( ;cdi < vecend; ++cdi){ next_delta_info(src->dstream + cdi->dso, &dc); - *out_bytes += DC_count_encode_bytes(&dc); - if (cdisize < size) { - size -= cdisize; + if (dc.ts < size) { + num_bytes += DC_count_encode_bytes(&dc); + size -= dc.ts; } else { + dc.ts = size; + num_bytes += DC_count_encode_bytes(&dc); size = 0; break; } } - *out_bytes += 0; - return num_dc; + assert(size == 0); + return num_bytes; } // Write a slice as defined by its absolute offset in bytes and its size into the given -// destination memory. The individual chunks written will be a deep copy of the source -// data chunks +// destination memory. The individual chunks written will be a byte copy of the source +// data chunk stream // Return: number of chunks in the slice inline -uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { - /* - assert(DIV_lbound(src) <= ofs); - assert((ofs + size) <= DIV_last(src)->to + src->di_last_size); + assert(DIV_lbound(src) <= tofs); + assert((tofs + size) <= DIV_last(src)->to + src->di_last_size); - DeltaInfo* cdc = DIV_closest_chunk(src, ofs); + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + + DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; // partial overlap - if (cdc->to != ofs) { - const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); - cdc += 1; - size -= dest->ts; - dest += 1; // must be here, we are reading the size ! + if (cdi->to != tofs) { + const uint relofs = tofs - cdi->to; + next_delta_info(src->dstream + cdi->dso, &dc); + const uint cdisize = dc.ts; + const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + + size -= actual_size; + + // adjust dc proportions + + DC_encode_to(&dc, &dest, relofs, actual_size); + num_chunks += 1; + cdi += 1; if (size == 0){ return num_chunks; @@ -563,14 +581,18 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull } const DeltaInfo* vecend = DIV_end(src); - for( ;(cdc < vecend) && size; ++cdc) + for( ;cdi < vecend; ++cdi) { num_chunks += 1; - if (cdc->ts < size) { - DC_copy_to(cdc, dest++); - size -= cdc->ts; + next_delta_info(src->dstream + cdi->dso, &dc); + if (dc.ts < size) { + // Full copy would be possible, but the final length of the dstream + // needs to be used as well to know how many bytes to copy + // TODO: make a DIV_ function for this + DC_encode_to(&dc, &dest, 0, dc.ts); + size -= dc.ts; } else { - DC_offset_copy_to(cdc, dest, 0, size); + DC_encode_to(&dc, &dest, 0, size); size = 0; break; } @@ -578,36 +600,44 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull assert(size == 0); return num_chunks; - */ - assert(0); // TODO - return 0; } + // Take slices of div into the corresponding area of the tsi, which is the topmost // delta to apply. bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { assert(tsi->num_chunks); - uint *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(uint)); + typedef struct { + uint bofs; // byte-offset of delta stream + uint dofs; // delta stream offset relative to tsi->cstart + } OffsetInfo; + + + OffsetInfo *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(OffsetInfo)); if (!offset_array){ return 0; } - uint* pofs = offset_array; - uint num_addchunks = 0; + OffsetInfo* pofs = offset_array; uint num_addbytes = 0; const uchar* data = TSI_first(tsi); + const uchar* prev_data = data; const uchar const* dend = TSI_end(tsi); + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); // OFFSET RUN - for (;data < dend; pofs++) + for (;data < dend; pofs++, prev_data = data) { + + pofs->bofs = num_addbytes; + pofs->dofs = (uint)(prev_data - data); + // Data chunks don't need processing - *pofs = num_addchunks; data = next_delta_info(data, &dc); if (dc.data){ @@ -615,49 +645,48 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } // offset the next chunk by the amount of chunks in the slice - // - 1, because we replace our own chunk - num_addchunks += DIV_count_slice_chunks_and_bytes(div, dc.so, dc.ts, &num_addbytes) - 1; - assert(num_addbytes); + // - N, because we replace our own chunk's bytes + num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } - /* - // reserve enough memory to hold all the new chunks - // reinit pointers, array could have been reallocated - TSI_resize(tsis, tsi->tdslen + num_addbytes); - dc = DIV_last(tdcv); - dcend = DIV_first(tdcv) - 1; - // now, that we have our pointers with the old size - tdcv->size += num_addchunks; + + // reserve enough memory to hold all the new chunks + TSI_resize(tsi, tsi->tdslen + num_addbytes); + const OffsetInfo const* pofs_start = offset_array - 1; + const OffsetInfo* cpofs; + uchar* ds; // pointer into the delta stream + const uchar* nds; // next pointer, used for size retrieving the size + uint num_addchunks = 0; // total amount of chunks added // Insert slices, from the end to the beginning, which allows memcpy // to be used, with a little help of the offset array - for (pofs -= 1; dc > dcend; dc--, pofs-- ) + for (cpofs = pofs - 1; cpofs > pofs_start; cpofs--) { + ds = (uchar*)(tsi->cstart + cpofs->dofs); + nds = next_delta_info(ds, &dc); + // Data chunks don't need processing - const uint ofs = *pofs; - if (dc->data){ + if (dc.data){ // NOTE: could peek the preceeding chunks to figure out whether they are // all just moved by ofs. In that case, they can move as a whole! // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort - if (ofs){ - memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaInfo)); + if (pofs->bofs){ + memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; } - // Copy Chunks, and move their target offset into place - // As we could override dc when slicing, we get the data here - const ull relofs = dc->to - dc->so; - - DeltaInfo* tdc = dc + ofs; - DeltaInfo* tdcend = tdc + DIV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); - for(;tdc < tdcend; tdc++){ - tdc->to += relofs; - } + // Copy Chunks - target offset is determined by their location and size + // hence it doesn't need specific adjustment + // -1 chunks because we overwrite our own chunk ( by not copying it ) + num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); + num_addchunks -= 1; } - */ + + tsi->num_chunks += num_addchunks; + PyMem_Free(offset_array); return 1; @@ -823,7 +852,7 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) if (cp_size == 0) cp_size = 0x10000; dc->to += dc->ts; - dc->data = 0; + dc->data = NULL; dc->so = cp_off; dc->ts = cp_size; From c16de40fccf84d066194d2bf95dea56aa76dafd1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 14:34:00 +0200 Subject: [PATCH 0110/3719] Implemented apply - there are still some issues to work out though --- _delta_apply.c | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 5f1d9c379..6d7c4c9d0 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -165,7 +165,6 @@ ull DC_rbound(const DeltaChunk* dc) } // Apply -// TODO: remove, just left it for reference inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) { @@ -252,7 +251,7 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) // DELTA INFO ///////////// typedef struct { - uint dso; // delta stream offset + uint dso; // delta stream offset, relative to the very start of the stream uint to; // target offset (cache) } DeltaInfo; @@ -281,10 +280,6 @@ int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) return 1; } - if (num_dc - vec->reserved_size < 10){ - num_dc += gDIV_grow_by; - } - #ifdef DEBUG bool was_null = vec->mem == NULL; #endif @@ -529,6 +524,8 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { + // TODO: could just count size of the delta chunk in the stream instead + // of reencoding num_bytes += DC_count_encode_bytes(&dc); size -= dc.ts; } else { @@ -731,7 +728,7 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { - /* + PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -749,23 +746,24 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) return NULL; } - const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DIV_end(&self->vec); - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + const uchar* base; + Py_ssize_t baselen; + PyObject_AsReadBuffer(pybuf, (const void**)&base, &baselen); PyObject* tmpargs = PyTuple_New(1); - for(; i < end; i++){ - DC_apply(i, data, writeproc, tmpargs); + const uchar* data = TSI_first(&self->istream); + const uchar const* dend = TSI_end(&self->istream); + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + + while (data < dend){ + data = next_delta_info(data, &dc); + DC_apply(&dc, base, writeproc, tmpargs); } Py_DECREF(tmpargs); - */ - // TODO - assert(0); Py_RETURN_NONE; } @@ -911,7 +909,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaInfoVector div; ToplevelStreamInfo tdsinfo; TSI_init(&tdsinfo); - DIV_init(&div, 100); // should be enough to keep the average text file + DIV_init(&div, 0); // GET TOPLEVEL DELTA STREAM @@ -1020,13 +1018,17 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; } + #ifdef DEBUG + fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + #endif + if (!DIV_connect_with_base(&tdsinfo, &div)){ error = 1; } #ifdef DEBUG - fprintf(stderr, "tdsinfo->len = %i\n", (int)tdsinfo.tdslen); - fprintf(stderr, "div->size = %i, div->reserved_size = %i\n", (int)div.size, (int)div.reserved_size); + fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif // destroy members, but keep memory From dc85535de371762503496313f5e753d61997dcfd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 16:22:59 +0200 Subject: [PATCH 0111/3719] Its much closer to a working state now, but still not quite there. Probably just a small thing --- _delta_apply.c | 72 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 6d7c4c9d0..65e949a04 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -57,6 +57,7 @@ void TSI_init(ToplevelStreamInfo* info) void TSI_destroy(ToplevelStreamInfo* info) { + fprintf(stderr, "TSI_destroy: %p\n", info); if (info->parent_object){ Py_DECREF(info->parent_object); info->parent_object = 0; @@ -164,6 +165,12 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } +inline +void DC_print(const DeltaChunk* dc, const char* prefix) +{ + fprintf(stderr, "%s-dc: to = %i, ts = %i, so = %i, data = %p\n", prefix, (int)dc->to, dc->ts, dc->so, dc->data); +} + // Apply inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) @@ -179,6 +186,8 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec assert(0); } + DC_print(dc, "DC_apply"); + // tuple steals reference, and will take care about the deallocation PyObject_Call(writer, tmpargs, NULL); @@ -233,14 +242,14 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) ull so = dc->so; // offset - c += so & 0x000000FF; - c += so & 0x0000FF00; - c += so & 0x00FF0000; - c += so & 0xFF000000; + c += (so & 0x000000FF) > 0; + c += (so & 0x0000FF00) > 0; + c += (so & 0x00FF0000) > 0; + c += (so & 0xFF000000) > 0; // size - max size is 0x10000, its encoded with 0 size bits - c += ts & 0x000000FF; - c += ts & 0x0000FF00; + c += (ts & 0x000000FF) > 0; + c += (ts & 0x0000FF00) > 0; return c; } @@ -413,7 +422,7 @@ void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG - fprintf(stderr, "Freeing %p\n", (void*)vec->mem); + fprintf(stderr, "DIV_destroy: %p\n", (void*)vec->mem); #endif PyMem_Free(vec->mem); vec->size = 0; @@ -494,8 +503,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) uint num_bytes = 0; DeltaInfo* cdi = DIV_closest_chunk(src, ofs); - - DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -503,13 +510,13 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) if (cdi->to != ofs) { const ull relofs = ofs - cdi->to; const uint cdisize = DIV_info_size(src, cdi); - const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= actual_size; + const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; + size -= max_size; // get the size in bytes the info would have next_delta_info(src->dstream + cdi->dso, &dc); dc.so += relofs; - dc.ts = actual_size; + dc.ts = max_size; num_bytes += DC_count_encode_bytes(&dc); cdi += 1; @@ -547,8 +554,9 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { + fprintf(stderr, "copy slice: ofs = %i, size = %i\n", (int)tofs, size); assert(DIV_lbound(src) <= tofs); - assert((tofs + size) <= DIV_last(src)->to + src->di_last_size); + assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -556,18 +564,22 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; +#ifdef DEBUG + const uchar* deststart = dest; +#endif + // partial overlap if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; next_delta_info(src->dstream + cdi->dso, &dc); const uint cdisize = dc.ts; - const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= actual_size; + size -= max_size; // adjust dc proportions - DC_encode_to(&dc, &dest, relofs, actual_size); + DC_encode_to(&dc, &dest, relofs, max_size); num_chunks += 1; cdi += 1; @@ -580,6 +592,7 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s const DeltaInfo* vecend = DIV_end(src); for( ;cdi < vecend; ++cdi) { + fprintf(stderr, "copy slice: cdi: to = %i, dso = %i\n", (int)cdi->to, (int)cdi->dso); num_chunks += 1; next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { @@ -595,6 +608,10 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s } } +#ifdef DEBUG + fprintf(stderr, "copy slice: Wrote %i bytes\n", (int)(dest - deststart)); +#endif + assert(size == 0); return num_chunks; } @@ -619,6 +636,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) OffsetInfo* pofs = offset_array; uint num_addbytes = 0; + uint dofs = 0; const uchar* data = TSI_first(tsi); const uchar* prev_data = data; @@ -627,16 +645,19 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); + // OFFSET RUN for (;data < dend; pofs++, prev_data = data) { - pofs->bofs = num_addbytes; - pofs->dofs = (uint)(prev_data - data); - - // Data chunks don't need processing data = next_delta_info(data, &dc); + pofs->dofs = dofs; + dofs += (uint)(data-prev_data); + + fprintf(stderr, "pofs->bofs = %i, ->dofs = %i\n", pofs->bofs, pofs->dofs); + DC_print(&dc, "count-run"); + // Data chunks don't need processing if (dc.data){ continue; } @@ -646,6 +667,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } + fprintf(stderr, "num_addbytes = %i\n", num_addbytes); + assert(DC_rbound(&dc) == tsi->target_size); // reserve enough memory to hold all the new chunks @@ -655,6 +678,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) uchar* ds; // pointer into the delta stream const uchar* nds; // next pointer, used for size retrieving the size uint num_addchunks = 0; // total amount of chunks added + DC_init(&dc, 0, 0, 0, NULL); // Insert slices, from the end to the beginning, which allows memcpy // to be used, with a little help of the offset array @@ -682,6 +706,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addchunks -= 1; } + fprintf(stderr, "num_addchunks = %i\n", num_addchunks); tsi->num_chunks += num_addchunks; PyMem_Free(offset_array); @@ -860,6 +885,8 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) dc->data = data; dc->ts = cmd; dc->so = 0; + + data += cmd; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); return NULL; @@ -993,8 +1020,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); // parse command stream - DeltaChunk dc; DeltaInfo* di = 0; // temporary pointer + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); @@ -1019,7 +1046,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #ifdef DEBUG + fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif if (!DIV_connect_with_base(&tdsinfo, &div)){ @@ -1028,7 +1057,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif // destroy members, but keep memory From c077f3f9ac56aa1019273c7ea548ae047e5974f6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 16:28:28 +0200 Subject: [PATCH 0112/3719] now it appears to work completely, still plenty of debug printing though --- _delta_apply.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_delta_apply.c b/_delta_apply.c index 65e949a04..350a2493d 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -693,7 +693,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // all just moved by ofs. In that case, they can move as a whole! // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort - if (pofs->bofs){ + if (cpofs->bofs){ memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; From 28263c97bca171d84b4f4ea022d6638076b903a4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 17:32:31 +0200 Subject: [PATCH 0113/3719] When using it with deeper chains, it can still crash as it can actually (try to) shrink a chunk. This is currently not handled. In that case, we had to virtually move everything x bytes, which should be much like an offset --- _delta_apply.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 350a2493d..2632fbb27 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -57,7 +57,10 @@ void TSI_init(ToplevelStreamInfo* info) void TSI_destroy(ToplevelStreamInfo* info) { +#ifdef DEBUG fprintf(stderr, "TSI_destroy: %p\n", info); +#endif + if (info->parent_object){ Py_DECREF(info->parent_object); info->parent_object = 0; @@ -129,6 +132,10 @@ bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) if (num_bytes <= info->tdslen){ return 1; } + +#ifdef DEBUG + fprintf(stderr, "TSI_resize: to %i bytes\n", num_bytes); +#endif uint ofs = (uint)(info->cstart - info->tds); info->tds = PyMem_Realloc((void*)info->tds, num_bytes); info->tdslen = num_bytes; @@ -186,7 +193,6 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec assert(0); } - DC_print(dc, "DC_apply"); // tuple steals reference, and will take care about the deallocation PyObject_Call(writer, tmpargs, NULL); @@ -554,7 +560,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { - fprintf(stderr, "copy slice: ofs = %i, size = %i\n", (int)tofs, size); assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); @@ -564,10 +569,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; -#ifdef DEBUG - const uchar* deststart = dest; -#endif - // partial overlap if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; @@ -592,7 +593,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s const DeltaInfo* vecend = DIV_end(src); for( ;cdi < vecend; ++cdi) { - fprintf(stderr, "copy slice: cdi: to = %i, dso = %i\n", (int)cdi->to, (int)cdi->dso); num_chunks += 1; next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { @@ -608,10 +608,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s } } -#ifdef DEBUG - fprintf(stderr, "copy slice: Wrote %i bytes\n", (int)(dest - deststart)); -#endif - assert(size == 0); return num_chunks; } @@ -624,7 +620,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->num_chunks); typedef struct { - uint bofs; // byte-offset of delta stream + int bofs; // byte-offset of delta stream uint dofs; // delta stream offset relative to tsi->cstart } OffsetInfo; @@ -635,7 +631,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } OffsetInfo* pofs = offset_array; - uint num_addbytes = 0; + int num_addbytes = 0; uint dofs = 0; const uchar* data = TSI_first(tsi); @@ -651,12 +647,10 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { pofs->bofs = num_addbytes; data = next_delta_info(data, &dc); + assert(data); pofs->dofs = dofs; dofs += (uint)(data-prev_data); - fprintf(stderr, "pofs->bofs = %i, ->dofs = %i\n", pofs->bofs, pofs->dofs); - DC_print(&dc, "count-run"); - // Data chunks don't need processing if (dc.data){ continue; @@ -667,7 +661,12 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } - fprintf(stderr, "num_addbytes = %i\n", num_addbytes); + /* + uint i = 0; + for (; i < tsi->num_chunks; i++){ + fprintf(stderr, "%i: bofs: %i, dofs: %i\n", i, offset_array[i].bofs, offset_array[i].dofs); + } + */ assert(DC_rbound(&dc) == tsi->target_size); @@ -695,6 +694,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // not worth the extra effort if (cpofs->bofs){ memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); + // memmove((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; } @@ -706,7 +706,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addchunks -= 1; } - fprintf(stderr, "num_addchunks = %i\n", num_addchunks); tsi->num_chunks += num_addchunks; PyMem_Free(offset_array); From 1a57dc133ec31c409819b2b40866529ed95a555b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 17:53:12 +0200 Subject: [PATCH 0114/3719] Well, the virtual movement doesn't work - the algorithm really worked only with a fixed chunk size. Now the only chance we have is to allocate an appropriately sized buffer, and work through it directly. This makes things easier, and will make things work \! --- _delta_apply.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 2632fbb27..9df0191e4 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -620,7 +620,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->num_chunks); typedef struct { - int bofs; // byte-offset of delta stream + uint bofs; // byte-offset of delta stream uint dofs; // delta stream offset relative to tsi->cstart } OffsetInfo; @@ -631,7 +631,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } OffsetInfo* pofs = offset_array; - int num_addbytes = 0; + uint num_addbytes = 0; + int bytes = 0; uint dofs = 0; const uchar* data = TSI_first(tsi); @@ -658,15 +659,16 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // offset the next chunk by the amount of chunks in the slice // - N, because we replace our own chunk's bytes - num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); + bytes = DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); + // if we shrink in size, compensate this by moving the start virtually + // + if (bytes < 0){ + fprintf(stderr, "hit negative bytes: %i\n", bytes); + tsi->cstart += abs(bytes); + } + num_addbytes += abs(bytes); } - /* - uint i = 0; - for (; i < tsi->num_chunks; i++){ - fprintf(stderr, "%i: bofs: %i, dofs: %i\n", i, offset_array[i].bofs, offset_array[i].dofs); - } - */ assert(DC_rbound(&dc) == tsi->target_size); @@ -701,8 +703,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // Copy Chunks - target offset is determined by their location and size // hence it doesn't need specific adjustment - // -1 chunks because we overwrite our own chunk ( by not copying it ) num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); + // -1 chunks because we overwrite our own chunk ( by not copying it ) num_addchunks -= 1; } From a7892bc7ebc99d2fe726287cc69716a3494b5ea2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 20:25:01 +0200 Subject: [PATCH 0115/3719] goooosh, it took so long to find a tiny nasty bug ... aarggghhh, lots of debug printing still in there, ... this one better be faster than anything else \! --- _delta_apply.c | 182 ++++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 9df0191e4..71d5b5f8c 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -15,6 +15,7 @@ typedef uchar bool; const ull gDIV_grow_by = 100; + // DELTA STREAM ACCESS /////////////////////// inline @@ -63,10 +64,14 @@ void TSI_destroy(ToplevelStreamInfo* info) if (info->parent_object){ Py_DECREF(info->parent_object); - info->parent_object = 0; + info->parent_object = NULL; } else if (info->tds){ PyMem_Free((void*)info->tds); } + info->tds = NULL; + info->cstart = NULL; + info->tdslen = 0; + info->num_chunks = 0; } inline @@ -122,26 +127,21 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) return 1; } -// make sure we have the given amount of memory available. This will change -// our official length in bytes right away, its up to the caller -// to do something useful with the freed space -// Return true on success -bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) +// Transfer ownership of the given stream into our instance. The amount of chunks +// remains the same, and needs to be set by the caller +void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) { - assert(info->tds); - if (num_bytes <= info->tdslen){ - return 1; - } + assert(info->parent_object == 0); + fprintf(stderr, "TSI_replace_stream\n"); -#ifdef DEBUG - fprintf(stderr, "TSI_resize: to %i bytes\n", num_bytes); -#endif uint ofs = (uint)(info->cstart - info->tds); - info->tds = PyMem_Realloc((void*)info->tds, num_bytes); - info->tdslen = num_bytes; + if (info->tds){ + PyMem_Free((void*)info->tds); + } + info->tds = stream; info->cstart = info->tds + ofs; + info->tdslen = streamlen; - return info->tds != NULL; } // DELTA CHUNK @@ -156,6 +156,9 @@ typedef struct { const uchar* data; } DeltaChunk; +// forward declarations +const uchar* next_delta_info(const uchar*, DeltaChunk*); + inline void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) { @@ -208,6 +211,8 @@ inline void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; + DC_print(dc, "DC_encode_to"); + fprintf(stderr, "DC_encode_to: ofs = %i, size = %i\n" , ofs, size); if (dc->data){ *out++ = (uchar)size; memcpy(out, dc->data+ofs, size); @@ -233,6 +238,18 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) *op = i; } + +#ifdef DEBUG + DeltaChunk mdc; + DC_init(&mdc, 0, 0, 0, NULL); + next_delta_info(*pout, &mdc); + assert(mdc.ts == size); + if (mdc.data) + assert(mdc.data); + else + assert(mdc.so == dc->so+ofs); +#endif + *pout = out; } @@ -497,8 +514,6 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) return DIV_last(vec); } -// forward declaration -const uchar* next_delta_info(const uchar*, DeltaChunk*); // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded @@ -558,10 +573,11 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) // data chunk stream // Return: number of chunks in the slice inline -uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) { assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); + fprintf(stderr, "copy_slice: ofs = %i, size = %i\n", (int)tofs, size); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -573,14 +589,12 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; next_delta_info(src->dstream + cdi->dso, &dc); - const uint cdisize = dc.ts; - const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; + const uint max_size = dc.ts - relofs < size ? dc.ts - relofs : size; size -= max_size; // adjust dc proportions - - DC_encode_to(&dc, &dest, relofs, max_size); + DC_encode_to(&dc, dest, relofs, max_size); num_chunks += 1; cdi += 1; @@ -599,10 +613,10 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s // Full copy would be possible, but the final length of the dstream // needs to be used as well to know how many bytes to copy // TODO: make a DIV_ function for this - DC_encode_to(&dc, &dest, 0, dc.ts); + DC_encode_to(&dc, dest, 0, dc.ts); size -= dc.ts; } else { - DC_encode_to(&dc, &dest, 0, size); + DC_encode_to(&dc, dest, 0, size); size = 0; break; } @@ -619,98 +633,90 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { assert(tsi->num_chunks); - typedef struct { - uint bofs; // byte-offset of delta stream - uint dofs; // delta stream offset relative to tsi->cstart - } OffsetInfo; - - - OffsetInfo *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(OffsetInfo)); - if (!offset_array){ - return 0; - } - - OffsetInfo* pofs = offset_array; - uint num_addbytes = 0; - int bytes = 0; - uint dofs = 0; + uint num_bytes = 0; const uchar* data = TSI_first(tsi); - const uchar* prev_data = data; - const uchar const* dend = TSI_end(tsi); + const uchar* dend = TSI_end(tsi); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); - // OFFSET RUN - for (;data < dend; pofs++, prev_data = data) + // COMPUTE SIZE OF TARGET STREAM + ///////////////////////////////// + for (;data < dend;) { - pofs->bofs = num_addbytes; data = next_delta_info(data, &dc); - assert(data); - pofs->dofs = dofs; - dofs += (uint)(data-prev_data); + DC_print(&dc, "count"); // Data chunks don't need processing if (dc.data){ + num_bytes += 1 + dc.ts; continue; } - // offset the next chunk by the amount of chunks in the slice - // - N, because we replace our own chunk's bytes - bytes = DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); - // if we shrink in size, compensate this by moving the start virtually - // - if (bytes < 0){ - fprintf(stderr, "hit negative bytes: %i\n", bytes); - tsi->cstart += abs(bytes); - } - num_addbytes += abs(bytes); + num_bytes += DIV_count_slice_bytes(div, dc.so, dc.ts); } - assert(DC_rbound(&dc) == tsi->target_size); - // reserve enough memory to hold all the new chunks - TSI_resize(tsi, tsi->tdslen + num_addbytes); - const OffsetInfo const* pofs_start = offset_array - 1; - const OffsetInfo* cpofs; - uchar* ds; // pointer into the delta stream - const uchar* nds; // next pointer, used for size retrieving the size - uint num_addchunks = 0; // total amount of chunks added + // GET NEW DELTA BUFFER + //////////////////////// + uchar *const dstream = PyMem_Malloc(num_bytes); + if (!dstream){ + return 0; + } + + + data = TSI_first(tsi); + const uchar *ndata = data; + dend = TSI_end(tsi); + + uint num_chunks = 0; + uchar* ds = dstream; DC_init(&dc, 0, 0, 0, NULL); - // Insert slices, from the end to the beginning, which allows memcpy - // to be used, with a little help of the offset array - for (cpofs = pofs - 1; cpofs > pofs_start; cpofs--) + // pick slices from the delta and put them into the new stream + for (; data < dend; data = ndata) { - ds = (uchar*)(tsi->cstart + cpofs->dofs); - nds = next_delta_info(ds, &dc); + ndata = next_delta_info(data, &dc); + + DC_print(&dc, "slice"); // Data chunks don't need processing if (dc.data){ - // NOTE: could peek the preceeding chunks to figure out whether they are - // all just moved by ofs. In that case, they can move as a whole! - // tests showed that this is very rare though, even in huge deltas, so its - // not worth the extra effort - if (cpofs->bofs){ - memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); - // memmove((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); - } + // just copy it over + memcpy((void*)ds, (void*)data, ndata - data); + ds += ndata - data; + num_chunks += 1; continue; } - // Copy Chunks - target offset is determined by their location and size - // hence it doesn't need specific adjustment - num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); - // -1 chunks because we overwrite our own chunk ( by not copying it ) - num_addchunks -= 1; + // Copy Chunks + num_chunks += DIV_copy_slice_to(div, &ds, dc.so, dc.ts); } + assert(ds - dstream == num_bytes); + assert(num_chunks >= tsi->num_chunks); + assert(DC_rbound(&dc) == tsi->target_size); + + // finally, replace the streams + TSI_replace_stream(tsi, dstream, num_bytes); + tsi->cstart = dstream; // we have NO header ! + assert(tsi->tds == dstream); + tsi->num_chunks = num_chunks; - tsi->num_chunks += num_addchunks; +#ifdef DEBUG + data = TSI_first(tsi); + dend = TSI_end(tsi); + + DC_init(&dc, 0, 0, 0, NULL); + + while (data < dend){ + data = next_delta_info(data, &dc); + DC_print(&dc, "debug"); + } +#endif - PyMem_Free(offset_array); return 1; } @@ -754,6 +760,7 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + fprintf(stderr, "DCL_apply\n"); PyObject* pybuf = 0; PyObject* writeproc = 0; @@ -890,6 +897,7 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) data += cmd; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + assert(0); return NULL; } @@ -1048,7 +1056,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i, target_size = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen, (int)tdsinfo.target_size); fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif From 81a96250f1758844a1c95f0293083c18c4ce98dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 20:37:39 +0200 Subject: [PATCH 0116/3719] Removed all debug code, it now runs about as fast as the previous version, but with less memory, still slower than the brute force version though --- _delta_apply.c | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 71d5b5f8c..068d89a17 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -132,7 +132,6 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) { assert(info->parent_object == 0); - fprintf(stderr, "TSI_replace_stream\n"); uint ofs = (uint)(info->cstart - info->tds); if (info->tds){ @@ -211,8 +210,6 @@ inline void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; - DC_print(dc, "DC_encode_to"); - fprintf(stderr, "DC_encode_to: ofs = %i, size = %i\n" , ofs, size); if (dc->data){ *out++ = (uchar)size; memcpy(out, dc->data+ofs, size); @@ -239,17 +236,6 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) *op = i; } -#ifdef DEBUG - DeltaChunk mdc; - DC_init(&mdc, 0, 0, 0, NULL); - next_delta_info(*pout, &mdc); - assert(mdc.ts == size); - if (mdc.data) - assert(mdc.data); - else - assert(mdc.so == dc->so+ofs); -#endif - *pout = out; } @@ -577,7 +563,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint { assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); - fprintf(stderr, "copy_slice: ofs = %i, size = %i\n", (int)tofs, size); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -647,7 +632,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) for (;data < dend;) { data = next_delta_info(data, &dc); - DC_print(&dc, "count"); // Data chunks don't need processing if (dc.data){ @@ -681,8 +665,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { ndata = next_delta_info(data, &dc); - DC_print(&dc, "slice"); - // Data chunks don't need processing if (dc.data){ // just copy it over @@ -705,17 +687,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->tds == dstream); tsi->num_chunks = num_chunks; -#ifdef DEBUG - data = TSI_first(tsi); - dend = TSI_end(tsi); - - DC_init(&dc, 0, 0, 0, NULL); - - while (data < dend){ - data = next_delta_info(data, &dc); - DC_print(&dc, "debug"); - } -#endif return 1; @@ -760,8 +731,6 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { - fprintf(stderr, "DCL_apply\n"); - PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -1056,8 +1025,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i, target_size = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen, (int)tdsinfo.target_size); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); + fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i KiB, target_size = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000, (int)tdsinfo.target_size/1000); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i KiB\n", (int)div.size, (int)div.reserved_size, (int)dlen/1000); #endif if (!DIV_connect_with_base(&tdsinfo, &div)){ @@ -1065,7 +1034,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #ifdef DEBUG - fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000); #endif // destroy members, but keep memory From ca829e0b341dd5c3ae1408b24702f2c75db6ec73 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 21:12:42 +0200 Subject: [PATCH 0117/3719] now byte-copying a few chunks where possible when slicing, which gives a few percent of performance --- _delta_apply.c | 20 +++++++++----------- stream.py | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 068d89a17..e99a803be 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -534,13 +534,12 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) } const DeltaInfo const* vecend = DIV_end(src); + const uchar* nstream; for( ;cdi < vecend; ++cdi){ - next_delta_info(src->dstream + cdi->dso, &dc); + nstream = next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { - // TODO: could just count size of the delta chunk in the stream instead - // of reencoding - num_bytes += DC_count_encode_bytes(&dc); + num_bytes += nstream - (src->dstream + cdi->dso); size -= dc.ts; } else { dc.ts = size; @@ -589,16 +588,15 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint } } - const DeltaInfo* vecend = DIV_end(src); - for( ;cdi < vecend; ++cdi) + const uchar* dstream = src->dstream + cdi->dso; + const uchar* nstream = dstream; + for( ; nstream; dstream = nstream) { num_chunks += 1; - next_delta_info(src->dstream + cdi->dso, &dc); + nstream = next_delta_info(dstream, &dc); if (dc.ts < size) { - // Full copy would be possible, but the final length of the dstream - // needs to be used as well to know how many bytes to copy - // TODO: make a DIV_ function for this - DC_encode_to(&dc, dest, 0, dc.ts); + memcpy(*dest, dstream, nstream - dstream); + *dest += nstream - dstream; size -= dc.ts; } else { DC_encode_to(&dc, dest, 0, size); diff --git a/stream.py b/stream.py index 0d8972898..f522bd318 100644 --- a/stream.py +++ b/stream.py @@ -445,7 +445,7 @@ def _set_cache_brute_(self, attr): #{ Configuration - if not has_perf_mod: + if has_perf_mod: _set_cache_ = _set_cache_brute_ else: _set_cache_ = _set_cache_too_slow_without_c From 21e2b5db748f18f70f16cd94ee9fd5adf16ce433 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 21:31:41 +0200 Subject: [PATCH 0118/3719] Updated algorithm paper with the latest changes - thats it for now --- doc/source/algorithm.rst | 18 ++++++++---------- test/performance/test_pack.py | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index 55207b67b..4374cb820 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -58,10 +58,10 @@ GitDB's reverse delta aggregation algorithm =========================================== The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. -In the current implementation, delta streams are parsed into DeltaChunks (->**DC**), which are kept in vectors. Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**). Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. -During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream. +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream.. The merging works by following a set of rules: * Merge into the top-level delta from the youngest ancestor delta to the oldest one @@ -72,10 +72,9 @@ The merging works by following a set of rules: * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. -Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. Here it is already possible to stream the result, which is feasible only if the memory of the base buffer + the memory of the TDS are smaller than a full size target buffer. Streaming will always make sense if the peak resulting from having the base, target and TDS buffers in memory together is unaffordable. -The memory consumption during the TDS processing are the uncompressed delta-bytes, the parsed DS, as well as the TDS. Afterwards one requires an allocated base buffer, the target buffer, as well as the TDS. -It is clearly visible that the current implementation does not at all reduce memory consumption, but the opposite is true as the TDS can be large for large files. +The memory consumption during the TDS processing is only the condensed delta-bytes, for each ADS an additional index is required which costs 8 byte per DC. When applying the TDS, one requires an allocated base buffer too.The target buffer can be allocated, but may be a writer as well. Performance Results ------------------- @@ -83,17 +82,16 @@ The benchmarking context was the same as for the brute-force GitDB algorithm. Th The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. -The throughput reaches 16.7 MiB/s, which equals 1344 streams/s, which makes it 15 times faster than the pure python version, and amazingly even 1.5 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. +The throughput reaches 15.2 MiB/s, which equals 1221 streams/s, which makes it nearly 14 times faster than the pure python version, and amazingly even 1.35 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. -All this comes at a relatively high memory consumption.Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. - -A 125 MB file took 3.1 seconds to unpack for instance, which is only 33% slower than the c implementation of the brute-force algorithm. +A 125 MB file took 2.5 seconds to unpack for instance, which is only 20% slower than the c implementation of the brute-force algorithm. Future work =========== -The current implementation of the reverse delta aggregation algorithm is already working well and fast, but leaves room for improvement in the realm of its memory consumption. One way to considerably reduce it would be to index the delta stream to determine bounds, instead of parsing it into a separate data structure Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. + +A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index f9169ffb0..32890dcb3 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -13,7 +13,7 @@ class TestPackedDBPerformance(TestBigRepoR): - def test_pack_random_access(self): + def _test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # sha lookup @@ -61,7 +61,7 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - def _disabled_test_correctness(self): + def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" From 2ddc5bad224d8f545ef3bb2ab3df98dfe063c5b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 12 Nov 2010 18:49:01 +0100 Subject: [PATCH 0119/3719] Updated to latest revision of async to support new thread-shutdown functionality --- ext/async | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/async b/ext/async index 5992bb6c8..7298742e1 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 5992bb6c85973ed81c54c71fef42e2413cd29e88 +Subproject commit 7298742e1b7236f2a4e369f915722a6618cef736 From 2a048f43d89112ff1f78ee05b59a9663e981f63f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 18 Nov 2010 23:42:14 +0100 Subject: [PATCH 0120/3719] Changed name/id of async submodule to something that doesn't look like a path --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 45ddc0b4c..9adc6121b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "ext/async"] +[submodule "async"] path = ext/async url = git://gitorious.org/git-python/async.git From 479ac8efc6aa6c579cba48bbb87f85f1cd0654f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Nov 2010 22:51:56 +0100 Subject: [PATCH 0121/3719] bumped version to 0.5.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e50e0fe7..6ebef0fa7 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.1", + version = "0.5.2", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", From dd4704095dbdc2770b9289b491bb166fa1d36a8d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Nov 2010 22:59:24 +0100 Subject: [PATCH 0122/3719] Updated changelog for 0.5.2 --- doc/source/changes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 84738ae05..7b8ebecc6 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -1,6 +1,12 @@ ######### Changelog ######### + +***** +0.5.2 +***** +* Improved performance of the c implementation, which now uses reverse-delta-aggregation to make a memory bound operation CPU bound. + ***** 0.5.1 ***** From 5d91a53372caa54cf7110cfa7fe1166956edc9c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 11:37:20 +0100 Subject: [PATCH 0123/3719] setup: added missing _delta_apply.c file to setup script, allowing the performance module to be compiled --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ebef0fa7..c7d0bd81c 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c'])], + ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 6edc28d6ab8b3f88673fd701b2c40104825a95df Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 11:54:00 +0100 Subject: [PATCH 0124/3719] Added delta_apply.h file to make more native use of python's build system, which should hopefully fix the easy_install trouble --- MANIFEST.in | 2 ++ _delta_apply.c | 4 +++- _delta_apply.h | 6 ++++++ _fun.c | 2 +- setup.py | 3 ++- 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 _delta_apply.h diff --git a/MANIFEST.in b/MANIFEST.in index a01acc452..7693cabf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,8 @@ include AUTHORS include README include _fun.c +include _delta_apply.c +include _delta_apply.h graft test diff --git a/_delta_apply.c b/_delta_apply.c index e99a803be..96ab30af9 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -1,10 +1,12 @@ -#include +#include "_delta_apply.h" #include #include #include #include #include + + typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; diff --git a/_delta_apply.h b/_delta_apply.h new file mode 100644 index 000000000..3e7e5f926 --- /dev/null +++ b/_delta_apply.h @@ -0,0 +1,6 @@ +#include + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams); +static PyObject* apply_delta(PyObject* self, PyObject* args); + +static PyTypeObject DeltaChunkListType; diff --git a/_fun.c b/_fun.c index befee4ec4..49970386f 100644 --- a/_fun.c +++ b/_fun.c @@ -1,5 +1,5 @@ #include -#include "_delta_apply.c" +#include "_delta_apply.h" static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { diff --git a/setup.py b/setup.py index c7d0bd81c..3a95ac845 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def run(self): except Exception: print "Ignored failure when building extensions, pure python modules will be used instead" # END ignore errors + def get_data_files(self): """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, @@ -77,7 +78,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'])], + ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 1bc281d31b8d31fd4dcbcd9b441b5c7b2c1b0bb5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 13:09:09 +0100 Subject: [PATCH 0125/3719] Added zip_safe flag to setup.py --- ext/async | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/async b/ext/async index 7298742e1..eccf3d63c 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 7298742e1b7236f2a4e369f915722a6618cef736 +Subproject commit eccf3d63c655e7a184ba339d747c98e965c1a63b diff --git a/setup.py b/setup.py index 3a95ac845..d235c91df 100755 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ def get_data_files(self): package_dir = {'gitdb':''}, ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], license = "BSD License", + zip_safe=False, requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', long_description = """GitDB is a pure-Python git object database""" From 9f977b8baaf9cbe9b38f3bdf4887cef5370b2229 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 25 Nov 2010 17:04:55 +0100 Subject: [PATCH 0126/3719] Switched async submodule to using github instead of gitorious --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9adc6121b..42efc2ed6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "async"] path = ext/async - url = git://gitorious.org/git-python/async.git + url = git://github.com/Byron/async.git From c0d5448fcd5ed6427ea96be01ef2d3ee509f3924 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:09:53 +0100 Subject: [PATCH 0127/3719] Ajusted all links to point to new repository on github --- README => README.rst | 5 ++--- doc/source/intro.rst | 5 ++--- ext/async | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) rename README => README.rst (84%) diff --git a/README b/README.rst similarity index 84% rename from README rename to README.rst index 33a566d86..753eb701c 100644 --- a/README +++ b/README.rst @@ -15,8 +15,7 @@ SOURCE ====== The source is available in a git repository at gitorious and github: -git://gitorious.org/git-python/gitdb.git -git://github.com/Byron/gitdb.git +git://github.com/gitpython-developers/gitdb.git Once the clone is complete, please be sure to initialize the submodules using @@ -33,7 +32,7 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -http://byronimo.lighthouseapp.com/projects/51787-gitpython +https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4d675cbc4..8fc0ec098 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -29,10 +29,9 @@ It is advised to have a look at the :ref:`Usage Guide ` for a br ================= Source Repository ================= -The latest source can be cloned using git from one of the following locations: +The latest source can be cloned using git from github: - * git://gitorious.org/git-python/gitdb.git - * git://github.com/Byron/gitdb.git + * git://github.com/gitpython-developers/gitdb.git License Information =================== diff --git a/ext/async b/ext/async index eccf3d63c..89790abd2 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit eccf3d63c655e7a184ba339d747c98e965c1a63b +Subproject commit 89790abd29bb4be851095b2f2c4c624b896b6e20 From 9fbc59da76b15cecb1ee37a8e48617fab58a077c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:24:07 +0100 Subject: [PATCH 0128/3719] moved all relevant files into the gitdb folder. Submodule relinked to point to new github location, and moved as well --- .gitmodules | 4 ++-- __init__.py => gitdb/__init__.py | 0 _delta_apply.c => gitdb/_delta_apply.c | 0 _delta_apply.h => gitdb/_delta_apply.h | 0 _fun.c => gitdb/_fun.c | 0 base.py => gitdb/base.py | 0 {db => gitdb/db}/__init__.py | 0 {db => gitdb/db}/base.py | 0 {db => gitdb/db}/git.py | 0 {db => gitdb/db}/loose.py | 0 {db => gitdb/db}/mem.py | 0 {db => gitdb/db}/pack.py | 0 {db => gitdb/db}/ref.py | 0 exc.py => gitdb/exc.py | 0 {ext => gitdb/ext}/async | 0 fun.py => gitdb/fun.py | 0 pack.py => gitdb/pack.py | 0 stream.py => gitdb/stream.py | 0 {test => gitdb/test}/__init__.py | 0 {test => gitdb/test}/db/__init__.py | 0 {test => gitdb/test}/db/lib.py | 0 {test => gitdb/test}/db/test_git.py | 0 {test => gitdb/test}/db/test_loose.py | 0 {test => gitdb/test}/db/test_mem.py | 0 {test => gitdb/test}/db/test_pack.py | 0 {test => gitdb/test}/db/test_ref.py | 0 .../7b/b839852ed5e3a069966281bb08d50012fb309b | Bin ...ack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx | Bin ...ck-11fdfa9e156ab73caae3b6da867192221f2089c2.pack | Bin ...ack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx | Bin ...ck-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack | Bin ...ack-c0438c19fb16422b6bbcce24387b3264416d485b.idx | Bin ...ck-c0438c19fb16422b6bbcce24387b3264416d485b.pack | Bin {test => gitdb/test}/lib.py | 0 {test => gitdb/test}/performance/lib.py | 0 {test => gitdb/test}/performance/test_pack.py | 0 .../test}/performance/test_pack_streaming.py | 0 {test => gitdb/test}/performance/test_stream.py | 0 {test => gitdb/test}/test_base.py | 0 {test => gitdb/test}/test_example.py | 0 {test => gitdb/test}/test_pack.py | 0 {test => gitdb/test}/test_stream.py | 0 {test => gitdb/test}/test_util.py | 0 typ.py => gitdb/typ.py | 0 util.py => gitdb/util.py | 0 45 files changed, 2 insertions(+), 2 deletions(-) rename __init__.py => gitdb/__init__.py (100%) rename _delta_apply.c => gitdb/_delta_apply.c (100%) rename _delta_apply.h => gitdb/_delta_apply.h (100%) rename _fun.c => gitdb/_fun.c (100%) rename base.py => gitdb/base.py (100%) rename {db => gitdb/db}/__init__.py (100%) rename {db => gitdb/db}/base.py (100%) rename {db => gitdb/db}/git.py (100%) rename {db => gitdb/db}/loose.py (100%) rename {db => gitdb/db}/mem.py (100%) rename {db => gitdb/db}/pack.py (100%) rename {db => gitdb/db}/ref.py (100%) rename exc.py => gitdb/exc.py (100%) rename {ext => gitdb/ext}/async (100%) rename fun.py => gitdb/fun.py (100%) rename pack.py => gitdb/pack.py (100%) rename stream.py => gitdb/stream.py (100%) rename {test => gitdb/test}/__init__.py (100%) rename {test => gitdb/test}/db/__init__.py (100%) rename {test => gitdb/test}/db/lib.py (100%) rename {test => gitdb/test}/db/test_git.py (100%) rename {test => gitdb/test}/db/test_loose.py (100%) rename {test => gitdb/test}/db/test_mem.py (100%) rename {test => gitdb/test}/db/test_pack.py (100%) rename {test => gitdb/test}/db/test_ref.py (100%) rename {test => gitdb/test}/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b (100%) rename {test => gitdb/test}/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack (100%) rename {test => gitdb/test}/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack (100%) rename {test => gitdb/test}/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack (100%) rename {test => gitdb/test}/lib.py (100%) rename {test => gitdb/test}/performance/lib.py (100%) rename {test => gitdb/test}/performance/test_pack.py (100%) rename {test => gitdb/test}/performance/test_pack_streaming.py (100%) rename {test => gitdb/test}/performance/test_stream.py (100%) rename {test => gitdb/test}/test_base.py (100%) rename {test => gitdb/test}/test_example.py (100%) rename {test => gitdb/test}/test_pack.py (100%) rename {test => gitdb/test}/test_stream.py (100%) rename {test => gitdb/test}/test_util.py (100%) rename typ.py => gitdb/typ.py (100%) rename util.py => gitdb/util.py (100%) diff --git a/.gitmodules b/.gitmodules index 42efc2ed6..9f06f7d07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "async"] - path = ext/async - url = git://github.com/Byron/async.git + path = gitdb/ext/async + url = git://github.com/gitpython-developers/async.git diff --git a/__init__.py b/gitdb/__init__.py similarity index 100% rename from __init__.py rename to gitdb/__init__.py diff --git a/_delta_apply.c b/gitdb/_delta_apply.c similarity index 100% rename from _delta_apply.c rename to gitdb/_delta_apply.c diff --git a/_delta_apply.h b/gitdb/_delta_apply.h similarity index 100% rename from _delta_apply.h rename to gitdb/_delta_apply.h diff --git a/_fun.c b/gitdb/_fun.c similarity index 100% rename from _fun.c rename to gitdb/_fun.c diff --git a/base.py b/gitdb/base.py similarity index 100% rename from base.py rename to gitdb/base.py diff --git a/db/__init__.py b/gitdb/db/__init__.py similarity index 100% rename from db/__init__.py rename to gitdb/db/__init__.py diff --git a/db/base.py b/gitdb/db/base.py similarity index 100% rename from db/base.py rename to gitdb/db/base.py diff --git a/db/git.py b/gitdb/db/git.py similarity index 100% rename from db/git.py rename to gitdb/db/git.py diff --git a/db/loose.py b/gitdb/db/loose.py similarity index 100% rename from db/loose.py rename to gitdb/db/loose.py diff --git a/db/mem.py b/gitdb/db/mem.py similarity index 100% rename from db/mem.py rename to gitdb/db/mem.py diff --git a/db/pack.py b/gitdb/db/pack.py similarity index 100% rename from db/pack.py rename to gitdb/db/pack.py diff --git a/db/ref.py b/gitdb/db/ref.py similarity index 100% rename from db/ref.py rename to gitdb/db/ref.py diff --git a/exc.py b/gitdb/exc.py similarity index 100% rename from exc.py rename to gitdb/exc.py diff --git a/ext/async b/gitdb/ext/async similarity index 100% rename from ext/async rename to gitdb/ext/async diff --git a/fun.py b/gitdb/fun.py similarity index 100% rename from fun.py rename to gitdb/fun.py diff --git a/pack.py b/gitdb/pack.py similarity index 100% rename from pack.py rename to gitdb/pack.py diff --git a/stream.py b/gitdb/stream.py similarity index 100% rename from stream.py rename to gitdb/stream.py diff --git a/test/__init__.py b/gitdb/test/__init__.py similarity index 100% rename from test/__init__.py rename to gitdb/test/__init__.py diff --git a/test/db/__init__.py b/gitdb/test/db/__init__.py similarity index 100% rename from test/db/__init__.py rename to gitdb/test/db/__init__.py diff --git a/test/db/lib.py b/gitdb/test/db/lib.py similarity index 100% rename from test/db/lib.py rename to gitdb/test/db/lib.py diff --git a/test/db/test_git.py b/gitdb/test/db/test_git.py similarity index 100% rename from test/db/test_git.py rename to gitdb/test/db/test_git.py diff --git a/test/db/test_loose.py b/gitdb/test/db/test_loose.py similarity index 100% rename from test/db/test_loose.py rename to gitdb/test/db/test_loose.py diff --git a/test/db/test_mem.py b/gitdb/test/db/test_mem.py similarity index 100% rename from test/db/test_mem.py rename to gitdb/test/db/test_mem.py diff --git a/test/db/test_pack.py b/gitdb/test/db/test_pack.py similarity index 100% rename from test/db/test_pack.py rename to gitdb/test/db/test_pack.py diff --git a/test/db/test_ref.py b/gitdb/test/db/test_ref.py similarity index 100% rename from test/db/test_ref.py rename to gitdb/test/db/test_ref.py diff --git a/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b similarity index 100% rename from test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b rename to gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx similarity index 100% rename from test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx rename to gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack b/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack similarity index 100% rename from test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack rename to gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx similarity index 100% rename from test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx rename to gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack b/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack similarity index 100% rename from test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack rename to gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx similarity index 100% rename from test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx rename to gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack similarity index 100% rename from test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack rename to gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack diff --git a/test/lib.py b/gitdb/test/lib.py similarity index 100% rename from test/lib.py rename to gitdb/test/lib.py diff --git a/test/performance/lib.py b/gitdb/test/performance/lib.py similarity index 100% rename from test/performance/lib.py rename to gitdb/test/performance/lib.py diff --git a/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py similarity index 100% rename from test/performance/test_pack.py rename to gitdb/test/performance/test_pack.py diff --git a/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py similarity index 100% rename from test/performance/test_pack_streaming.py rename to gitdb/test/performance/test_pack_streaming.py diff --git a/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py similarity index 100% rename from test/performance/test_stream.py rename to gitdb/test/performance/test_stream.py diff --git a/test/test_base.py b/gitdb/test/test_base.py similarity index 100% rename from test/test_base.py rename to gitdb/test/test_base.py diff --git a/test/test_example.py b/gitdb/test/test_example.py similarity index 100% rename from test/test_example.py rename to gitdb/test/test_example.py diff --git a/test/test_pack.py b/gitdb/test/test_pack.py similarity index 100% rename from test/test_pack.py rename to gitdb/test/test_pack.py diff --git a/test/test_stream.py b/gitdb/test/test_stream.py similarity index 100% rename from test/test_stream.py rename to gitdb/test/test_stream.py diff --git a/test/test_util.py b/gitdb/test/test_util.py similarity index 100% rename from test/test_util.py rename to gitdb/test/test_util.py diff --git a/typ.py b/gitdb/typ.py similarity index 100% rename from typ.py rename to gitdb/typ.py diff --git a/util.py b/gitdb/util.py similarity index 100% rename from util.py rename to gitdb/util.py From 0300d0d6c4af63257d094813ceaab2302906680c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:45:49 +0100 Subject: [PATCH 0129/3719] Fixed unittests --- doc/source/tutorial.rst | 2 +- gitdb/__init__.py | 8 +++++++- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_pack.py | 3 +-- gitdb/test/db/test_ref.py | 2 +- gitdb/test/test_example.py | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index cfe3fb284..55a737f6b 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -37,7 +37,7 @@ Both have two sets of methods, one of which allows interacting with single objec Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: - ldb = LooseObjectDB(fixture_path("../../.git/objects")) + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index d79788fc4..c8e77759e 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -6,7 +6,13 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext')) + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async')) + + try: + import async + except ImportError: + raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH") + #END verify import #} END initialization diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index d2ae10bad..bcab1fc55 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,7 +7,7 @@ class TestGitDB(TestDBBase): def test_reading(self): - gdb = GitDB(fixture_path('../../.git/objects')) + gdb = GitDB(fixture_path('../../../.git/objects')) # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 0386b3f80..1d0cb96e7 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -15,8 +15,7 @@ def test_writing(self, path): pdb = PackedDB(path) # on demand, we init our pack cache - num_packs = 2 - assert len(pdb.entities()) == num_packs + num_packs = len(pdb.entities()) assert pdb._st_mtime != 0 # test pack directory changed: diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 9df25cef6..af75016b0 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -34,7 +34,7 @@ def test_writing(self, path): # setup alternate file # add two, one is invalid - own_repo_path = fixture_path('../../.git/objects') # use own repo + own_repo_path = fixture_path('../../../.git/objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 3fc3fcf5e..2d6096132 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -11,7 +11,7 @@ class TestExamples(TestBase): def test_base(self): - ldb = LooseObjectDB(fixture_path("../../.git/objects")) + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) From 88d500edf2163be3b249ae288a06b725934560d9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:57:16 +0100 Subject: [PATCH 0130/3719] setup and doc generation works once again --- MANIFEST.in | 8 ++++---- doc/source/conf.py | 2 +- gitdb/fun.py | 9 +++++---- gitdb/pack.py | 2 +- setup.py | 7 +++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7693cabf8..b14aed9b3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,11 +4,11 @@ include CHANGES include AUTHORS include README -include _fun.c -include _delta_apply.c -include _delta_apply.h +include gitdb/_fun.c +include gitdb/_delta_apply.c +include gitdb/_delta_apply.h -graft test +prune gitdb/test global-exclude .git* global-exclude *.pyc diff --git a/doc/source/conf.py b/doc/source/conf.py index e10addb8d..28deb3106 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('../../../')) +sys.path.append(os.path.abspath('../../')) # -- General configuration ----------------------------------------------------- diff --git a/gitdb/fun.py b/gitdb/fun.py index 0b14f82ac..fc4172040 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -44,8 +44,7 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'reverse_connect_deltas', - 'connect_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList') #{ Structures @@ -492,8 +491,10 @@ def stream_copy(read, write, size, chunk_size): return dbw def connect_deltas(dstreams): - """Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" diff --git a/gitdb/pack.py b/gitdb/pack.py index 30da52c63..affcbe275 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -651,7 +651,7 @@ def is_valid_stream(self, sha, use_crc=False): :param use_crc: if True, the index' crc for the sha is used to determine :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is + whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the data of the actual undeltified object, as it depends on more than just this stream. diff --git a/setup.py b/setup.py index d235c91df..3c6617422 100755 --- a/setup.py +++ b/setup.py @@ -75,10 +75,9 @@ def get_data_files(self): author_email = "byronimo@gmail.com", url = "http://gitorious.org/git-python/gitdb", packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), - package_data={'gitdb' : ['AUTHORS', 'README'], - 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, - package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], + package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + package_dir = {'gitdb':'gitdb'}, + ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, requires=('async (>=0.6.1)',), From 1fe2a9403fafa810af25062c7e6b20be8e6be480 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 1 Dec 2010 10:28:14 +0100 Subject: [PATCH 0131/3719] setup .gitmodules to use a trackin branch automatically --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 9f06f7d07..3db4c676d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "async"] path = gitdb/ext/async url = git://github.com/gitpython-developers/async.git + branch = master From 6d315b8a92ae2cba936bd38e433592383f42cc10 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 23 Feb 2011 00:26:14 +0100 Subject: [PATCH 0132/3719] Added license information file --- LICENSE | 30 ++++++++++++++++++++++++++++++ gitdb/ext/async | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..be11e73c1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the GitDB project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/gitdb/ext/async b/gitdb/ext/async index 89790abd2..e91b46928 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 89790abd29bb4be851095b2f2c4c624b896b6e20 +Subproject commit e91b4692825e0121504262ba58ac0b2bcd09fea3 From df570f00f611073a20796128ca167474aa7826fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 23 Feb 2011 00:43:31 +0100 Subject: [PATCH 0133/3719] preprended all modules with licensing information --- gitdb/__init__.py | 4 ++++ gitdb/base.py | 4 ++++ gitdb/db/__init__.py | 4 ++++ gitdb/db/base.py | 4 ++++ gitdb/db/git.py | 4 ++++ gitdb/db/loose.py | 4 ++++ gitdb/db/mem.py | 4 ++++ gitdb/db/pack.py | 4 ++++ gitdb/db/ref.py | 4 ++++ gitdb/exc.py | 4 ++++ gitdb/ext/async | 2 +- gitdb/fun.py | 4 ++++ gitdb/pack.py | 4 ++++ gitdb/stream.py | 4 ++++ gitdb/test/__init__.py | 4 ++++ gitdb/test/db/__init__.py | 4 ++++ gitdb/test/db/lib.py | 4 ++++ gitdb/test/db/test_git.py | 4 ++++ gitdb/test/db/test_loose.py | 4 ++++ gitdb/test/db/test_mem.py | 4 ++++ gitdb/test/db/test_pack.py | 4 ++++ gitdb/test/db/test_ref.py | 4 ++++ gitdb/test/lib.py | 4 ++++ gitdb/test/performance/lib.py | 4 ++++ gitdb/test/performance/test_pack.py | 4 ++++ gitdb/test/performance/test_pack_streaming.py | 4 ++++ gitdb/test/performance/test_stream.py | 4 ++++ gitdb/test/test_base.py | 4 ++++ gitdb/test/test_example.py | 4 ++++ gitdb/test/test_pack.py | 4 ++++ gitdb/test/test_stream.py | 4 ++++ gitdb/test/test_util.py | 4 ++++ gitdb/typ.py | 4 ++++ gitdb/util.py | 4 ++++ 34 files changed, 133 insertions(+), 1 deletion(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c8e77759e..a551f37dc 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Initialize the object database module""" import sys diff --git a/gitdb/base.py b/gitdb/base.py index d0bcc0866..ff1062bf6 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( bin_to_hex, diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index 85a0a6874..e5935b7c2 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import * from loose import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 1914dbbce..2189d4193 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( pool, diff --git a/gitdb/db/git.py b/gitdb/db/git.py index f0e63b15a..b8fc46aa0 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( CompoundDB, ObjectDBW, diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 521be44c2..6cd1cefd5 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( FileDBBase, ObjectDBR, diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index f361ab801..8012ad15e 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains the MemoryDatabase implementation""" from loose import LooseObjectDB from base import ( diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 0ec8a4e3b..eef3f712e 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" from base import ( FileDBBase, diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index c149c03d0..898984323 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( CompoundDB, ) diff --git a/gitdb/exc.py b/gitdb/exc.py index 012cdbc6b..96fa874e3 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with common exceptions""" from util import to_hex_sha diff --git a/gitdb/ext/async b/gitdb/ext/async index e91b46928..10310824c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit e91b4692825e0121504262ba58ac0b2bcd09fea3 +Subproject commit 10310824c001deab8fea85b88ebda0696f964b3e diff --git a/gitdb/fun.py b/gitdb/fun.py index fc4172040..3e035dbf1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains basic c-functions which usually contain performance critical code Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" diff --git a/gitdb/pack.py b/gitdb/pack.py index affcbe275..09e7defc5 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( BadObject, diff --git a/gitdb/stream.py b/gitdb/stream.py index 0d8972898..6c3b8d31f 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from cStringIO import StringIO import errno diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 0dec7750f..760f531be 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php import gitdb.util diff --git a/gitdb/test/db/__init__.py b/gitdb/test/db/__init__.py index e69de29bb..8a681e428 100644 --- a/gitdb/test/db/__init__.py +++ b/gitdb/test/db/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 0080d919e..416c8c588 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index bcab1fc55..310116351 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.exc import BadObject from gitdb.db import GitDB diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 8e8a9bfc3..ee2d78d08 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import LooseObjectDB from gitdb.exc import BadObject diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 4a9b7ee12..188cb0a93 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( MemoryDB, diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 1d0cb96e7..e8ba6f8fc 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import PackedDB from gitdb.test.lib import fixture_path diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index af75016b0..0d8eeebb3 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ReferenceDB diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 3fb87d547..342234adc 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( OStream, diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 45e0ca53f..761113d51 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os from gitdb.test.lib import * diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 32890dcb3..da952b17a 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" from lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 4d47cdfcc..22a62a39d 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" from lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 1afc1a1a0..f5f2e2e4d 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" from lib import TestBigRepoR from gitdb.db import * diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 740e50bcd..1b20faf87 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( TestBase, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 2d6096132..753177560 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" from lib import * from gitdb import IStream diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 770a78bad..928f0cd97 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" from lib import ( TestBase, diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 948cbe766..523f77056 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( TestBase, diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 6a389d27c..90f4156b9 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" import tempfile import os diff --git a/gitdb/typ.py b/gitdb/typ.py index 54a1f84be..e84dd2455 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" #{ String types diff --git a/gitdb/util.py b/gitdb/util.py index 1ea182025..4bb3c7352 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php import binascii import os import mmap From 3bcb30f1916239f1af25b4d6d8934c64bb47f8ea Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 11:06:36 +0200 Subject: [PATCH 0134/3719] Added qt creator project as it has advantages regarding the navigation over jEdit, although jedit has advantages regarding the syntax highlighting and whitespace visualization --- gitdb.pro | 48 +++++++++ gitdb.pro.user | 267 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 gitdb.pro create mode 100644 gitdb.pro.user diff --git a/gitdb.pro b/gitdb.pro new file mode 100644 index 000000000..a682615c4 --- /dev/null +++ b/gitdb.pro @@ -0,0 +1,48 @@ + +OTHER_FILES += \ + setup.py \ + README.rst \ + MANIFEST \ + Makefile \ + LICENSE \ + AUTHORS \ + gitdb/util.py \ + gitdb/typ.py \ + gitdb/stream.py \ + gitdb/pack.py \ + gitdb/__init__.py \ + gitdb/fun.py \ + gitdb/exc.py \ + gitdb/base.py \ + doc/source/tutorial.rst \ + doc/source/intro.rst \ + doc/source/index.rst \ + doc/source/conf.py \ + doc/source/changes.rst \ + doc/source/api.rst \ + doc/source/algorithm.rst \ + gitdb/db/ref.py \ + gitdb/db/pack.py \ + gitdb/db/mem.py \ + gitdb/db/loose.py \ + gitdb/db/__init__.py \ + gitdb/db/git.py \ + gitdb/db/base.py \ + gitdb/test/test_util.py \ + gitdb/test/test_stream.py \ + gitdb/test/test_pack.py \ + gitdb/test/test_example.py \ + gitdb/test/test_base.py \ + gitdb/test/lib.py \ + gitdb/test/__init__.py \ + gitdb/test/performance/test_stream.py \ + gitdb/test/performance/test_pack_streaming.py \ + gitdb/test/performance/test_pack.py \ + gitdb/test/performance/lib.py + +HEADERS += \ + gitdb/_delta_apply.h + +SOURCES += \ + gitdb/_fun.c \ + gitdb/_delta_apply.c diff --git a/gitdb.pro.user b/gitdb.pro.user new file mode 100644 index 000000000..398cb70a1 --- /dev/null +++ b/gitdb.pro.user @@ -0,0 +1,267 @@ + + + + ProjectExplorer.Project.ActiveTarget + 0 + + + ProjectExplorer.Project.EditorSettings + + Default + + + + ProjectExplorer.Project.Target.0 + + Desktop + + Qt4ProjectManager.Target.DesktopTarget + 0 + 0 + 1 + + + 0 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + false + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + 4 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + No deployment + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + gitdb + + Qt4ProjectManager.Qt4RunConfiguration + 2 + + gitdb.pro + false + false + + false + + 3768 + true + false + + + + /usr/bin/nosetests + -s + gitdb/test/test_pack.py + + 2 + python + false + + $BUILDDIR + Run python + test-pack + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + true + false + + 2 + + + + ProjectExplorer.Project.TargetCount + 1 + + + ProjectExplorer.Project.Updater.EnvironmentId + {d38778e3-6b24-4419-8fc3-c8cc320f55e0} + + + ProjectExplorer.Project.Updater.FileVersion + 8 + + From 810d1e38315c6e886c1daef93670840b213ee78a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 11:08:26 +0200 Subject: [PATCH 0135/3719] Added stub for pack writing implementation which should work for pack streaming over a transport as well --- gitdb/fun.py | 10 +--------- gitdb/pack.py | 25 +++++++++++++++++++------ gitdb/test/test_pack.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 3e035dbf1..34978cd97 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -411,15 +411,7 @@ def pack_object_header_info(data): size += (c & 0x7f) << s s += 7 # END character loop - - try: - return (type_id, size, i) - except KeyError: - # invalid object type - we could try to be smart now and decode part - # of the stream to get the info, problem is that we had trouble finding - # the exact start of the content stream - raise BadObjectType(type_id) - # END handle exceptions + return (type_id, size, i) def msb_size(data, offset=0): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index 09e7defc5..335fe3c00 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -98,8 +98,7 @@ def pack_object_at(data, offset, as_stream): # REF DELTA elif type_id == REF_DELTA: total_rela_offset = data_rela_offset+20 - ref_sha = data[data_rela_offset:total_rela_offset] - delta_info = ref_sha + delta_info = data[data_rela_offset:total_rela_offset] # BASE OBJECT else: # assume its a base object @@ -561,11 +560,10 @@ def _sha_to_index(self, sha): def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" - indexfile = self._index + _sha = self._index.sha _object = self._object - for index in xrange(indexfile.size()): - sha = indexfile.sha(index) - yield _object(sha, as_stream, index) + for index in xrange(self._index.size()): + yield _object(_sha(index), as_stream, index) # END for each index def _object(self, sha, as_stream, index=-1): @@ -760,5 +758,20 @@ def collect_streams(self, sha): return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + @classmethod + def create(cls, object_iter, pack_write, index_write=None): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :note: The destination of the write functions is up to the user. It could + be a socket, or a file for instance""" + + #} END interface diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 928f0cd97..8e98808c7 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -25,8 +25,12 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from itertools import izip +from itertools import izip, chain +from nose import SkipTest + import os +import sys +import tempfile #{ Utilities @@ -134,7 +138,9 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def test_pack_entity(self): + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_iterators = list(); for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): @@ -143,6 +149,7 @@ def test_pack_entity(self): entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile + pack_iterators.append(entity.stream_iter()) count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): @@ -174,9 +181,28 @@ def test_pack_entity(self): # END for each info, stream tuple assert count == size - # END for each entity + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path = tempfile.mktemp('', "pack", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + for pp, ip in ((pack_path, )*2, (index_path, None)): + pfile = open(pp, 'wb') + ifile = None + if ip: + ifile = open(ip, 'wb') + #END handle ip + + PackEntity.create(chain(*pack_iterators), pfile, ifile) + assert os.path.getsize(pp) > 100 + if ip is not None: + assert os.path.getsize(ip) > 100 + #END verify files exist + #END for each packpath, indexpath pair + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack - pass + raise SkipTest() From e83210d99aaac5768827c448909fa04d63776e64 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 18:01:05 +0200 Subject: [PATCH 0136/3719] initial version of pack writing, which seems to work, but still needs some more testing and verification --- gitdb/exc.py | 3 + gitdb/fun.py | 20 +++- gitdb/pack.py | 196 ++++++++++++++++++++++++++++++++++++++-- gitdb/stream.py | 18 +++- gitdb/test/test_pack.py | 51 ++++++++--- 5 files changed, 266 insertions(+), 22 deletions(-) diff --git a/gitdb/exc.py b/gitdb/exc.py index 96fa874e3..e087047b4 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -17,6 +17,9 @@ class BadObject(ODBError): def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) + +class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): """Thrown if a possibly shortened name does not uniquely represent a single object diff --git a/gitdb/fun.py b/gitdb/fun.py index 34978cd97..5bbe8efc3 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -48,7 +48,7 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures @@ -412,6 +412,24 @@ def pack_object_header_info(data): s += 7 # END character loop return (type_id, size, i) + +def create_pack_object_header(obj_type, obj_size): + """:return: string defining the pack header comprised of the object type + and its incompressed size in bytes + :parmam obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = str() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + return hdr def msb_size(data, offset=0): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index 335fe3c00..6c32949d6 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -5,7 +5,8 @@ """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( BadObject, - UnsupportedOperation + UnsupportedOperation, + ParseError ) from util import ( zlib, @@ -15,6 +16,7 @@ ) from fun import ( + create_pack_object_header, pack_object_header_info, is_equal_canonical_sha, type_id_to_type_map, @@ -47,6 +49,7 @@ DeltaApplyReader, Sha1Writer, NullStream, + FlexibleSha1Writer ) from struct import ( @@ -54,6 +57,8 @@ unpack, ) +from binascii import crc32 + from itertools import izip import array import os @@ -119,10 +124,113 @@ def pack_object_at(data, offset, as_stream): return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) # END handle info # END handle stream - + +def write_stream_to_pack(read, write, zstream, want_crc=False): + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param want_crc: if True, the crc will be generated over the compressed data. + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if want_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + crc = 0 + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + if len(chunk) != chunk_size: + break + #END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + return (br, bw, crc) + + #} END utilities +class IndexWriter(object): + """Utility to cache index information, allowing to write all information later + in one go to the given stream + :note: currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_binsha, write): + """Write the index file using the given write method + :param pack_binsha: sha over the whole pack that we index""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,)*256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[ord(t[0][0])] += 1 + #END prepare fanout + + for i in xrange(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i+1] = v + #END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1]&0xffffffff)) + #END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist)-1 + #END hande 64 bit offsets + sha_write(pack('>L', ofs&0xffffffff)) + #END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + #END for each offset + + # trailer + assert(len(pack_binsha) == 20) + sha_write(pack_binsha) + write(sha_writer.sha(as_hex=False)) + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find @@ -135,6 +243,8 @@ class PackIndexFile(LazyMixin): # used in v2 indices _sha_list_offset = 8 + 1024 + index_v2_signature = '\377tOc' + index_version_default = 2 def __init__(self, indexpath): super(PackIndexFile, self).__init__() @@ -155,7 +265,7 @@ def _set_cache_(self, attr): # to access the fanout table or related properties # CHECK VERSION - self._version = (self._data[:4] == '\377tOc' and 2) or 1 + self._version = (self._data[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: version_id = unpack_from(">L", self._data, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id @@ -383,6 +493,8 @@ class PackFile(LazyMixin): case""" __slots__ = ('_packpath', '_data', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 # offset into our data at which the first object starts first_object_offset = 3*4 # header bytes @@ -396,15 +508,19 @@ def _set_cache_(self, attr): self._data = file_contents_ro_filepath(self._packpath) # read the header information - type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) + type_id, self._version, self._size = unpack_from(">LLL", self._data, 0) # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? else: # must be '_size' or '_version' # read header info - we do that just with a file stream - type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12)) + type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12)) # END handle header + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + #END assert type id + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data @@ -759,7 +875,8 @@ def collect_streams(self, sha): @classmethod - def create(cls, object_iter, pack_write, index_write=None): + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. @@ -769,9 +886,74 @@ def create(cls, object_iter, pack_write, index_write=None): :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: binary sha over all the contents of the pack :note: The destination of the write functions is up to the user. It could - be a socket, or a file for instance""" + be a socket, or a file for instance + :note: writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + #END handle list type + object_count = len(objs) + #END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + #END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, want_crc = index_write) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + #END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + #END abort once we are done + #END for each object + + if actual_count != object_count: + raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + #END count assertion + + # write footer + binsha = pack_writer.sha(as_hex = False) + assert len(binsha) == 20 + pack_write(binsha) + ofs += len(binsha) # just for completeness ;) + + if wants_index: + index.write(binsha, index_write) + #END handle index + return binsha #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 6c3b8d31f..8010a0551 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,9 @@ except ImportError: pass -__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams @@ -557,6 +559,20 @@ def sha(self, as_hex = False): #} END interface +class FlexibleSha1Writer(Sha1Writer): + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) + + class ZippedStoreShaWriter(Sha1Writer): """Remembers everything someone writes to it and generates a sha""" __slots__ = ('buf', 'zip') diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 8e98808c7..c4d8df165 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -140,7 +140,7 @@ def test_pack(self): @with_rw_directory def test_pack_entity(self, rw_dir): - pack_iterators = list(); + pack_objs = list() for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): @@ -149,7 +149,7 @@ def test_pack_entity(self, rw_dir): entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile - pack_iterators.append(entity.stream_iter()) + pack_objs.extend(entity.stream_iter()) count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): @@ -182,24 +182,49 @@ def test_pack_entity(self, rw_dir): assert count == size # END for each entity - + # pack writing - write all packs into one # index path can be None pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) - for pp, ip in ((pack_path, )*2, (index_path, None)): - pfile = open(pp, 'wb') - ifile = None - if ip: - ifile = open(ip, 'wb') + iteration = 0 + for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + pfile = open(ppath, 'wb') + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write #END handle ip - PackEntity.create(chain(*pack_iterators), pfile, ifile) - assert os.path.getsize(pp) > 100 - if ip is not None: - assert os.path.getsize(ip) > 100 + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + for obj in pack_objs: + obj.stream.seek(0) + #END rewind streams + iteration += 1 + + binsha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pfile.close() + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == binsha + + # verify index + if ipath is not None: + assert os.path.getsize(ipath) > 100 + #END verify files exist - #END for each packpath, indexpath pair + + if ifile: + ifile.close() + #END handle index + #END for each packpath, indexpath pair + + # def test_pack_64(self): From 98a19ac1986b623277098263f01696827567c584 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 18:49:22 +0200 Subject: [PATCH 0137/3719] Implemented remainder of the test, and it already shows that something is wrong with my packs. Probably something stupid ;) --- gitdb/pack.py | 62 +++++++++++++++++++++++++++++++---------- gitdb/test/lib.py | 7 +++-- gitdb/test/test_pack.py | 34 +++++++++++++++------- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 6c32949d6..d90eeb991 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -12,6 +12,7 @@ zlib, LazyMixin, unpack_from, + bin_to_hex, file_contents_ro_filepath, ) @@ -60,6 +61,7 @@ from binascii import crc32 from itertools import izip +import tempfile import array import os import sys @@ -176,9 +178,10 @@ def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - def write(self, pack_binsha, write): + def write(self, pack_sha, write): """Write the index file using the given write method - :param pack_binsha: sha over the whole pack that we index""" + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) @@ -192,11 +195,10 @@ def write(self, pack_binsha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) - tmplist[i+1] = v + tmplist[i+1] += v #END write each fanout entry sha_write(pack('>L', tmplist[255])) @@ -226,9 +228,11 @@ def write(self, pack_binsha, write): #END for each offset # trailer - assert(len(pack_binsha) == 20) - sha_write(pack_binsha) - write(sha_writer.sha(as_hex=False)) + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha @@ -767,7 +771,9 @@ def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - :param use_crc: if True, the index' crc for the sha is used to determine + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. :param sha: 20 byte sha1 of the object whose stream to verify whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the @@ -890,7 +896,8 @@ def write_pack(cls, object_iter, pack_write, index_write=None, this would be the place to put it. Otherwise we have to pre-iterate and store all items into a list to get the number, which uses more memory than necessary. :param zlib_compression: the zlib compression level to use - :return: binary sha over all the contents of the pack + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None :note: The destination of the write functions is up to the user. It could be a socket, or a file for instance :note: writes only undeltified objects""" @@ -944,16 +951,41 @@ def write_pack(cls, object_iter, pack_write, index_write=None, #END count assertion # write footer - binsha = pack_writer.sha(as_hex = False) - assert len(binsha) == 20 - pack_write(binsha) - ofs += len(binsha) # just for completeness ;) + pack_sha = pack_writer.sha(as_hex = False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + index_sha = None if wants_index: - index.write(binsha, index_write) + index_sha = index.write(pack_sha, index_write) #END handle index - return binsha + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + :note: for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) #} END interface diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 342234adc..50645be65 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -42,19 +42,22 @@ def with_rw_directory(func): def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) + keep = False try: try: return func(self, path) except Exception: print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + keep = True raise finally: # Need to collect here to be sure all handles have been closed. It appears # a windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. - gc.collect() - shutil.rmtree(path) + if not keep: + gc.collect() + shutil.rmtree(path) # END handle exception # END wrapper diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index c4d8df165..e9c933e15 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -188,6 +188,10 @@ def test_pack_entity(self, rw_dir): pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + #END utility for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): pfile = open(ppath, 'wb') iwrite = None @@ -198,12 +202,11 @@ def test_pack_entity(self, rw_dir): # make sure we rewind the streams ... we work on the same objects over and over again if iteration > 0: - for obj in pack_objs: - obj.stream.seek(0) + rewind_streams() #END rewind streams iteration += 1 - binsha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) pfile.close() assert os.path.getsize(ppath) > 100 @@ -211,20 +214,31 @@ def test_pack_entity(self, rw_dir): pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default - assert pf.checksum() == binsha + assert pf.checksum() == pack_sha # verify index if ipath is not None: + ifile.close() assert os.path.getsize(ipath) > 100 - + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) #END verify files exist - - if ifile: - ifile.close() - #END handle index #END for each packpath, indexpath pair - # + # verify the packs throughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in reversed(range(2)): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + #END for each info + assert count == len(pack_objs) def test_pack_64(self): From 184a776960efdc2a83eac571c9c046ffcee3e7c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 20:27:44 +0200 Subject: [PATCH 0138/3719] crc needs to be done on the pack object header as well, of course --- gitdb/pack.py | 22 ++++++++++++++++++---- gitdb/test/test_pack.py | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index d90eeb991..7ae9786e6 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -127,15 +127,20 @@ def pack_object_at(data, offset, as_stream): # END handle info # END handle stream -def write_stream_to_pack(read, write, zstream, want_crc=False): +def write_stream_to_pack(read, write, zstream, base_crc=None): """Copy a stream as read from read function, zip it, and write the result. Count the number of written bytes and return it - :param want_crc: if True, the crc will be generated over the compressed data. - :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if want_crc + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc was false""" br = 0 # bytes read bw = 0 # bytes written + want_crc = base_crc is not None crc = 0 + if want_crc: + crc = base_crc + #END initialize crc while True: chunk = read(chunk_size) @@ -651,6 +656,9 @@ def __init__(self, pack_or_index_path): def _set_cache_(self, attr): # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" @@ -926,15 +934,21 @@ def write_pack(cls, object_iter, pack_write, index_write=None, actual_count = 0 for obj in objs: actual_count += 1 + crc = 0 # object header hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + #END handle crc pwrite(hdr) # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, want_crc = index_write) + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) assert(br == obj.size) if wants_index: index.append(obj.binsha, crc, ofs) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index e9c933e15..4a7f1caf2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -234,7 +234,7 @@ def rewind_streams(): count = 0 for info in entity.info_iter(): count += 1 - for use_crc in reversed(range(2)): + for use_crc in range(2): assert entity.is_valid_stream(info.binsha, use_crc) # END for each crc mode #END for each info From 0c7a3ec9829caa6632afd3e46901be67c63ae7fa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 23:40:04 +0200 Subject: [PATCH 0139/3719] Fixed _perf module, which built, but didn't link dynamically. All the time, I think it never successfully imported, but its hard to believe this slipped by. Added performance test for pack-writing, which isn't really showing what I want as it currently read data from a densly compressed pack which takes most of the time in the nearly pure python implementation. Compared to c++, all the measured performance is just below anything I'd want to use. But we shouldn't forget this is just a test implementation, writing packs is quite simple actually, if you leave out the delta compression part and the delta logic --- gitdb/_delta_apply.c | 19 ++++---- gitdb/_delta_apply.h | 6 +-- gitdb/test/performance/test_pack_streaming.py | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index 96ab30af9..f03e7ea6d 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -1,4 +1,4 @@ -#include "_delta_apply.h" +#include <_delta_apply.h> #include #include #include @@ -463,7 +463,7 @@ void DIV_reset(DeltaInfoVector* vec) // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! -static inline +inline DeltaInfo* DIV_append(DeltaInfoVector* vec) { if (vec->size + 1 > vec->reserved_size){ @@ -703,7 +703,7 @@ typedef struct { } DeltaChunkList; -static + int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) { if(args && PySequence_Size(args) > 0){ @@ -715,20 +715,20 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return 0; } -static + void DCL_dealloc(DeltaChunkList* self) { TSI_destroy(&(self->istream)); } -static + PyObject* DCL_py_rbound(DeltaChunkList* self) { return PyLong_FromUnsignedLongLong(self->istream.target_size); } // Write using a write function, taking remaining bytes from a base buffer -static + PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { PyObject* pybuf = 0; @@ -769,13 +769,13 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) Py_RETURN_NONE; } -static PyMethodDef DCL_methods[] = { +PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; -static PyTypeObject DeltaChunkListType = { +PyTypeObject DeltaChunkListType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "DeltaChunkList", /*tp_name*/ @@ -897,7 +897,7 @@ uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) return num_chunks; } -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator PyObject* stream_iter = 0; @@ -1088,7 +1088,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // Write using a write function, taking remaining bytes from a base buffer // replaces the corresponding method in python -static PyObject* apply_delta(PyObject* self, PyObject* args) { PyObject* pybbuf = 0; diff --git a/gitdb/_delta_apply.h b/gitdb/_delta_apply.h index 3e7e5f926..1fcd53832 100644 --- a/gitdb/_delta_apply.h +++ b/gitdb/_delta_apply.h @@ -1,6 +1,6 @@ #include -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams); -static PyObject* apply_delta(PyObject* self, PyObject* args); +extern PyObject* connect_deltas(PyObject *self, PyObject *dstreams); +extern PyObject* apply_delta(PyObject* self, PyObject* args); -static PyTypeObject DeltaChunkListType; +extern PyTypeObject DeltaChunkListType; diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 22a62a39d..795ed1e26 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -8,14 +8,57 @@ ) from gitdb.db.pack import PackedDB +from gitdb.stream import NullStream +from gitdb.pack import PackEntity import os import sys from time import time +from nose import SkipTest + +class CountedNullStream(NullStream): + __slots__ = '_bw' + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + class TestPackStreamingPerformance(TestBigRepoR): + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 5000 + count = 0 + total_size = 0 + st = time() + objs = list() + for sha in pdb.sha_iter(): + count += 1 + objs.append(pdb.stream(sha)) + if count == ni: + break + #END gather objects for pack-writing + elapsed = time() - st + print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + + st = time() + PackEntity.write_pack(objs, ostream.write) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + + def test_stream_reading(self): + raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # streaming only, meant for --with-profile runs From 03767cc17c6111c4c39cd4dde0517f5415515598 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 15:08:59 +0200 Subject: [PATCH 0140/3719] Initial setup for the testing framework; includes setup tools configuration and readme --- .gitignore | 6 +++++ Makefile | 37 +++++++++++++++++++++++++++++ README.rst | 50 +++++++++++++++++++++++++++++++++++++++ setup.py | 49 ++++++++++++++++++++++++++++++++++++++ smmap/__init__.py | 7 ++++++ smmap/mman.py | 3 +++ smmap/stream.py | 7 ++++++ smmap/test/__init__.py | 0 smmap/test/lib.py | 24 +++++++++++++++++++ smmap/test/test_mman.py | 7 ++++++ smmap/test/test_stream.py | 7 ++++++ 11 files changed, 197 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.rst create mode 100755 setup.py create mode 100644 smmap/__init__.py create mode 100644 smmap/mman.py create mode 100644 smmap/stream.py create mode 100644 smmap/test/__init__.py create mode 100644 smmap/test/lib.py create mode 100644 smmap/test/test_mman.py create mode 100644 smmap/test/test_stream.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..6cfb58df1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +build/ +.coverage +coverage +dist/ +MANIFEST diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ca3051dd4 --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +.PHONY: build sdist cover test clean-files clean-docs doc all + +all: + $(info Possible targets:) + $(info doc) + $(info clean-docs) + $(info clean-files) + $(info clean) + $(info test) + $(info coverage) + $(info build) + $(info sdist) + +doc: + cd docs && make html + +clean-docs: + cd docs && make clean + +clean-files: + git clean -fx + +clean: clean-files clean-docs + +test: + nosetests + +coverage: + nosetests --with-coverage --cover-package=smmap + +build: + ./setup.py build + +sdist: + ./setup.py sdist + + diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..08dc7111f --- /dev/null +++ b/README.rst @@ -0,0 +1,50 @@ +#################### +Sliding MMap (smmap) +#################### +A straight forward implementation of a slidinging memory map. +The idea is that every access to a file goes through a memory map manager, which will on demand map a region of a file and provide a string-like object for reading. + +When reading from it, you will have to check whether you are still within your window boundary, and possibly obtain a new window as required. + +The great benefit of this system is that you can use it to map files of any size even on 32 bit systems. Additionally it will be able to close unused windows right away to return system resources. If there are multiple clients for the same file and location, the same window will be reused as well. + +As there is a global management facility, you are also able to forcibly free all open handles which is handy on windows, which would otherwise prevent the deletion of the involved files. + +For convenience, a stream class is provided which hides the usage of the memory manager behind a simple stream interface. + +************ +LIMITATIONS +************ +The access is readonly by design. + +************ +REQUIREMENTS +************ +* Python 2.4 or higher + +******* +Install +******* +TODO + +****** +Source +****** +The source is available at git://github.com/Byron/smmap.git and can be cloned using:: + + git clone git://github.com/Byron/smmap.git + +************ +MAILING LIST +************ +http://groups.google.com/group/git-python + +************* +ISSUE TRACKER +************* +https://github.com/Byron/smmap/issues + +******* +LICENSE +******* +New BSD License diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..e2bb622fd --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import os +import codecs +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +import smmap + +if os.path.exists("README.rst"): + long_description = codecs.open('README.rst', "r", "utf-8").read() +else: + long_description = "See http://github.com/nvie/smmap/tree/master" + +setup( + name="smmap", + version=smmap.__version__, + description="A pure git implementation of a sliding window memory map manager", + author=smmap.__author__, + author_email=smmap.__contact__, + url=smmap.__homepage__, + platforms=["any"], + license="BSD", + packages=find_packages(), + zip_safe=True, + classifiers=[ + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", + #"Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Windows", + "Operating System :: OSX", + "Programming Language :: Python", + ], + long_description=long_description, +) diff --git a/smmap/__init__.py b/smmap/__init__.py new file mode 100644 index 000000000..82cff638c --- /dev/null +++ b/smmap/__init__.py @@ -0,0 +1,7 @@ +"""Intialize the smmap package""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/Byron/smmap" +version_info = (0, 8, 0) +__version__ = '.'.join(str(i) for i in version_info) diff --git a/smmap/mman.py b/smmap/mman.py new file mode 100644 index 000000000..dfa39db8c --- /dev/null +++ b/smmap/mman.py @@ -0,0 +1,3 @@ +"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" + +__all__ = [] diff --git a/smmap/stream.py b/smmap/stream.py new file mode 100644 index 000000000..bc5e568ea --- /dev/null +++ b/smmap/stream.py @@ -0,0 +1,7 @@ +"""Module with a simple stream implementation using the memory manager""" + +from mman import * + +__all__ = [] + + diff --git a/smmap/test/__init__.py b/smmap/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/smmap/test/lib.py b/smmap/test/lib.py new file mode 100644 index 000000000..450dd9dda --- /dev/null +++ b/smmap/test/lib.py @@ -0,0 +1,24 @@ +"""Provide base classes for the test system""" +from unittest import TestCase + +__all__ = ['TestBase'] + + +class TestBase(TestCase): + """Foundation used by all tests""" + + #{ Configuration + + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + #END overrides + + #{ Interface + + #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py new file mode 100644 index 000000000..43aef7771 --- /dev/null +++ b/smmap/test/test_mman.py @@ -0,0 +1,7 @@ +from lib import TestBase + +from smmap.mman import * + +class TestMMan(TestBase): + def test_basics(self): + assert False diff --git a/smmap/test/test_stream.py b/smmap/test/test_stream.py new file mode 100644 index 000000000..fae928d9a --- /dev/null +++ b/smmap/test/test_stream.py @@ -0,0 +1,7 @@ +from lib import TestBase + +from smmap.stream import * + +class TestStream(TestBase): + def test_basics(self): + assert False From b75a09b997e278e5351b60bc17b738cf62594fe0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 15:19:45 +0200 Subject: [PATCH 0141/3719] Added docs framework --- doc/.gitignore | 2 + doc/Makefile | 89 +++++++++++++++++++ doc/make.bat | 113 ++++++++++++++++++++++++ doc/source/changes.rst | 9 ++ doc/source/conf.py | 194 +++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 23 +++++ 6 files changed, 430 insertions(+) create mode 100644 doc/.gitignore create mode 100644 doc/Makefile create mode 100644 doc/make.bat create mode 100644 doc/source/changes.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..32060acdd --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +build +*.version_info diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..675ec2094 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/smmap.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/smmap.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 000000000..6900a2a46 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\smmap.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\smmap.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/doc/source/changes.rst b/doc/source/changes.rst new file mode 100644 index 000000000..ee17e0af0 --- /dev/null +++ b/doc/source/changes.rst @@ -0,0 +1,9 @@ +######### +Changelog +######### + +********** +v0.8.0 +********** + +- Initial Release diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..a0dac1166 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# smmap documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 8 15:14:25 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'smmap' +copyright = u'2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.8.0' +# The full version, including alpha/beta/rc tags. +release = '0.8.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'smmapdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'smmap.tex', u'smmap Documentation', + u'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..cb03044e0 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,23 @@ +.. smmap documentation master file, created by + sphinx-quickstart on Wed Jun 8 15:14:25 2011. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to smmap's documentation! +================================= +**smmap** is a pure python implementation of a sliding memory map to help unifying memory mapped access on 32 and 64 bit systems and to help managing resources more efficiently. + +Contents: + +.. toctree:: + :maxdepth: 2 + + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + From 2156e9ab02ea27289e6b26c11b024685b3816935 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 19:15:41 +0200 Subject: [PATCH 0142/3719] Implemented Window including test. Started Region implementation as well as test, but noticed that a critical feature, the mmap's offset, doesn't exist prior to python 2.6. Its total crap, so is python --- README.rst | 5 +- smmap/mman.py | 110 +++++++++++++++++++++++++++++++++++++++- smmap/test/lib.py | 43 +++++++++++++++- smmap/test/test_mman.py | 67 +++++++++++++++++++++++- 4 files changed, 219 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 08dc7111f..4c22eed2b 100644 --- a/README.rst +++ b/README.rst @@ -15,12 +15,13 @@ For convenience, a stream class is provided which hides the usage of the memory ************ LIMITATIONS ************ -The access is readonly by design. +* The access is readonly by design. +* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0. ************ REQUIREMENTS ************ -* Python 2.4 or higher +* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function. ******* Install diff --git a/smmap/mman.py b/smmap/mman.py index dfa39db8c..004987a8f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,3 +1,111 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -__all__ = [] +__all__ = ["MappedMemoryManager"] + +import os +import mmap + +from mmap import PAGESIZE + +#{ Utilities + +def align_to_page(num, round_up): + """Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / PAGESIZE) * PAGESIZE; + if round_up and (res != num): + res += PAGESIZE; + #END handle size + return res; + +#}END utilities + +class Window(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "Window(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region.ofs_begin(), region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + self.ofs = align_to_page(self.ofs, 0) + self.size = align_to_page(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class Region(object): + """Defines a mapped region of memory, aligned to pagesizes + :note: deallocates used region automatically on destruction""" + __slots__ = ( + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_nc', # number of clients using this region + '_uc' # total amount of usages + ) + + + def __init__(self, path, ofs, size): + """Initialize a region, allocate the memory map + :param path: path to the file to map + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._nc = 0 + self._uc = 0 + + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + try: + self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs) + finally: + os.close(fd) + #END close file handle + + +class MappedMemoryManager(object): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + :note: currently not thread-safe ! + :note: in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 450dd9dda..0605313d3 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -1,9 +1,50 @@ """Provide base classes for the test system""" from unittest import TestCase +import os +import tempfile -__all__ = ['TestBase'] +__all__ = ['TestBase', 'FileCreator'] +#{ Utilities + +class FileCreator(object): + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._path = tempfile.mktemp(prefix=prefix) + self._size = size + + fp = open(self._path, "wb") + fp.seek(size-1) + fp.write('1') + fp.close() + + assert os.path.getsize(self.path) == size + + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + #END exception handling + + + @property + def path(self): + return self._path + + @property + def size(self): + return self._size + +#} END utilities + class TestBase(TestCase): """Foundation used by all tests""" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 43aef7771..faee8c9cf 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,70 @@ -from lib import TestBase +from lib import TestBase, FileCreator from smmap.mman import * +from smmap.mman import Region +from smmap.mman import Window + +import sys +import mmap class TestMMan(TestBase): + + _window_test_size = 1000 * 1000 * 8 + 5195 + + def test_window(self): + wl = Window(0, 1) # left + wc = Window(1, 1) # center + wc2 = Window(10, 5) # another center + wr = Window(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 + + + def test_region(self): + fc = FileCreator(self._window_test_size, "window_test") + rfull = Region(fc.path, 0, fc.size) + + + + Window.from_region # todo + pass + def test_basics(self): - assert False + pass From 66a78db0127ed84e22cda5aed2009955765959e4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 20:26:51 +0200 Subject: [PATCH 0143/3719] Implemented MappedRegion type including test --- smmap/mman.py | 73 ++++++++++++++++++++++++++++++++++++++--- smmap/test/test_mman.py | 23 ++++++++++--- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 004987a8f..c5a388750 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -3,6 +3,7 @@ __all__ = ["MappedMemoryManager"] import os +import sys import mmap from mmap import PAGESIZE @@ -64,16 +65,22 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class Region(object): +class MappedRegion(object): """Defines a mapped region of memory, aligned to pagesizes :note: deallocates used region automatically on destruction""" - __slots__ = ( + __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_nc', # number of clients using this region - '_uc' # total amount of usages - ) + '_uc', # total amount of usages + '_ms' # actual size of the mapping + ] + _need_compat_layer = sys.version_info[1] < 6 + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + def __init__(self, path, ofs, size): """Initialize a region, allocate the memory map @@ -88,10 +95,66 @@ def __init__(self, path, ofs, size): fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: - self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs) + kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + corrected_size = size + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size, corrected_size), **kwargs) + + print len(self._mf) + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, size) + #END handle buffer wrapping finally: os.close(fd) #END close file handle + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return len(self._mf) + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self.size() + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + + def client_count(self): + """:return: number of clients currently using this region""" + return self._nc + + def adjust_client_count(self, ofs): + """Adjust the client count by the given positive or negative offset""" + self._nc += ofs + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def adjust_usage_count(self, ofs): + """Adjust the usage count by the given positive or negative offset""" + self._uc += ofs + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return len(self._mf) - self._b + + def ofs_end(self): + return len(self._mf) + #END handle compat layer class MappedMemoryManager(object): diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index faee8c9cf..482ef923c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import Region +from smmap.mman import MappedRegion from smmap.mman import Window import sys @@ -59,12 +59,27 @@ def test_window(self): def test_region(self): fc = FileCreator(self._window_test_size, "window_test") - rfull = Region(fc.path, 0, fc.size) + half_size = fc.size / 2 + rofs = 4000 + rfull = MappedRegion(fc.path, 0, fc.size) + rhalfofs = MappedRegion(fc.path, rofs, fc.size) + rhalfsize = MappedRegion(fc.path, 0, half_size) + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + + # window constructor + w = Window.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - Window.from_region # todo - pass def test_basics(self): pass From 0bf73877f860a86d9cf601154e9a9f76292e63c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 21:00:33 +0200 Subject: [PATCH 0144/3719] Fixed bug in test case as it didn't properly align its offset to a page --- smmap/mman.py | 20 +++++++++++++++----- smmap/test/test_mman.py | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index c5a388750..27904f5d0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -7,6 +7,7 @@ import mmap from mmap import PAGESIZE +from sys import getrefcount #{ Utilities @@ -71,7 +72,6 @@ class MappedRegion(object): __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) - '_nc', # number of clients using this region '_uc', # total amount of usages '_ms' # actual size of the mapping ] @@ -90,24 +90,24 @@ def __init__(self, path, ofs, size): allocated the the size automatically adjusted :raise Exception: if no memory can be allocated""" self._b = ofs - self._nc = 0 self._uc = 0 fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) corrected_size = size + sizeofs = ofs if self._need_compat_layer: del(kwargs['offset']) corrected_size += ofs + sizeofs = 0 # END handle python not supporting offset ! Arg # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size, corrected_size), **kwargs) + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) - print len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) #END handle buffer wrapping @@ -133,7 +133,8 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" - return self._nc + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 def adjust_client_count(self, ofs): """Adjust the client count by the given positive or negative offset""" @@ -157,6 +158,15 @@ def ofs_end(self): #END handle compat layer +class Cursor(object): + """Pointer into the mapped region of the memory manager, keeping the current window + alive until it is destroyed""" + + +class MappedRegionList(list): + """List of MappedRegion instances with specific functionality""" + + class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 482ef923c..00c9d90e8 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,8 +1,11 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MappedRegion +from smmap.mman import align_to_page from smmap.mman import Window +from smmap.mman import MappedRegion +from smmap.mman import MappedRegionList +from smmap.mman import Cursor import sys import mmap @@ -56,11 +59,10 @@ def test_window(self): wc.align() assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 - def test_region(self): fc = FileCreator(self._window_test_size, "window_test") half_size = fc.size / 2 - rofs = 4000 + rofs = align_to_page(4200, False) rfull = MappedRegion(fc.path, 0, fc.size) rhalfofs = MappedRegion(fc.path, rofs, fc.size) rhalfsize = MappedRegion(fc.path, 0, half_size) @@ -76,10 +78,20 @@ def test_region(self): assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + # window constructor w = Window.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + def test_region_list(self): + pass + + def test_cursor(self): + pass def test_basics(self): pass From 25e50356ab7c5392cece8132a8ee5b99c1789734 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 21:25:07 +0200 Subject: [PATCH 0145/3719] Added rather trivial implementation for the region list, including test --- smmap/mman.py | 30 ++++++++++++++++++++++++++---- smmap/test/test_mman.py | 7 ++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 27904f5d0..e0b4a5e64 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -158,15 +158,37 @@ def ofs_end(self): #END handle compat layer +class MappedRegionList(list): + """List of MappedRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path', # path which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MappedRegionList, cls).__new__(cls) + + def __init__(self, path): + self._path = path + self._file_size = None + + def path(self): + """:return: path to file whose regions we manage""" + return self._path + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + self._file_size = os.stat(self._path).st_size + #END update file size + return self._file_size + + class Cursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed""" - -class MappedRegionList(list): - """List of MappedRegion instances with specific functionality""" - class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 00c9d90e8..2027d9004 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -88,7 +88,12 @@ def test_region(self): assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): - pass + fc = FileCreator(100, "sample_file") + ml = MappedRegionList(fc.path) + + assert len(ml) == 0 + assert ml.path() == fc.path + assert ml.file_size() == fc.size def test_cursor(self): pass From cab0e3d6d995b7814e09157086d07c3ca2c901d4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 22:36:11 +0200 Subject: [PATCH 0146/3719] Moved all utility types into their own module --- smmap/mman.py | 204 ++++++---------------------------------- smmap/test/lib.py | 2 +- smmap/test/test_mman.py | 93 +----------------- smmap/test/test_util.py | 89 ++++++++++++++++++ smmap/util.py | 189 +++++++++++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 268 deletions(-) create mode 100644 smmap/test/test_util.py create mode 100644 smmap/util.py diff --git a/smmap/mman.py b/smmap/mman.py index e0b4a5e64..0ecc1aad1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,192 +1,46 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -__all__ = ["MappedMemoryManager"] +__all__ = ["MappedMemoryManager", "MemoryCursor"] -import os -import sys -import mmap - -from mmap import PAGESIZE -from sys import getrefcount - -#{ Utilities - -def align_to_page(num, round_up): - """Align the given integer number to the closest page offset, which usually is 4096 bytes. - :param round_up: if True, the next higher multiple of page size is used, otherwise - the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) - :return: num rounded to closest page""" - res = (num / PAGESIZE) * PAGESIZE; - if round_up and (res != num): - res += PAGESIZE; - #END handle size - return res; - -#}END utilities - -class Window(object): - """Utility type which is used to snap windows towards each other, and to adjust their size""" - __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes +from util import ( + MemoryWindow, + MappedRegion, + MappedRegionList, ) - def __init__(self, offset, size): - self.ofs = offset - self.size = size - - def __repr__(self): - return "Window(%i, %i)" % (self.ofs, self.size) - - @classmethod - def from_region(cls, region): - """:return: new window from a region""" - return cls(region.ofs_begin(), region.size()) - - def ofs_end(self): - return self.ofs + self.size - def align(self): - self.ofs = align_to_page(self.ofs, 0) - self.size = align_to_page(self.size, 1) - - def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, - but don't make yourself larger than max_size. - The resize will assure that the new window still contains the old window area""" - rofs = self.ofs - window.ofs_end() - nsize = rofs + self.size - rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs - self.size += rofs - - def extend_right_to(self, window, max_size): - """Adjust the size to make our window end where the right window begins, but don't - get larger than max_size""" - self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) - - -class MappedRegion(object): - """Defines a mapped region of memory, aligned to pagesizes - :note: deallocates used region automatically on destruction""" - __slots__ = [ - '_b' , # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_ms' # actual size of the mapping - ] - _need_compat_layer = sys.version_info[1] < 6 - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot - +class MemoryCursor(object): + """Pointer into the mapped region of the memory manager, keeping the current window + alive until it is destroyed""" + __slots__ = ( + '_manager', # the manger keeping all file regions + '_regions', # a regions list with regions for our file + '_region', # WEAK REF to our current region + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) - def __init__(self, path, ofs, size): - """Initialize a region, allocate the memory map - :param path: path to the file to map - :param ofs: **aligned** offset into the file to be mapped - :param size: if size is larger then the file on disk, the whole file will be - allocated the the size automatically adjusted - :raise Exception: if no memory can be allocated""" - self._b = ofs - self._uc = 0 - - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) - try: - kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) - corrected_size = size - sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will - # bark that the size is too large ... many extra file accesses because - # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, size) - #END handle buffer wrapping - finally: - os.close(fd) - #END close file handle - - def ofs_begin(self): - """:return: absolute byte offset to the first byte of the mapping""" - return self._b - - def size(self): - """:return: total size of the mapped region in bytes""" - return len(self._mf) + def __init__(self, manager = None, regions = None): + self._manager = manager + self._regions = regions + self._region = region + self._ofs = 0 + self._size = 0 - def ofs_end(self): - """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self.size() + def __del__(self): + self._destroy() - def includes_ofs(self, ofs): - """:return: True if the given offset can be read in our mapped region""" - return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + def _destroy(self): + """Destruction code to decrement counters""" - def client_count(self): - """:return: number of clients currently using this region""" - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" - def adjust_client_count(self, ofs): - """Adjust the client count by the given positive or negative offset""" - self._nc += ofs - - def usage_count(self): - """:return: amount of usages so far""" - return self._uc - - def adjust_usage_count(self, ofs): - """Adjust the usage count by the given positive or negative offset""" - self._uc += ofs - - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return len(self._mf) - self._b - - def ofs_end(self): - return len(self._mf) - #END handle compat layer + #{ Interface - -class MappedRegionList(list): - """List of MappedRegion instances associating a path with a list of regions.""" - __slots__ = ( - '_path', # path which is mapped by all our regions - '_file_size' # total size of the file we map - ) - - def __new__(cls, path): - return super(MappedRegionList, cls).__new__(cls) - def __init__(self, path): - self._path = path - self._file_size = None - - def path(self): - """:return: path to file whose regions we manage""" - return self._path - - def file_size(self): - """:return: size of file we manager""" - if self._file_size is None: - self._file_size = os.stat(self._path).st_size - #END update file size - return self._file_size + #} END interface - -class Cursor(object): - """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed""" class MappedMemoryManager(object): diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 0605313d3..6957dcab0 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -49,7 +49,7 @@ class TestBase(TestCase): """Foundation used by all tests""" #{ Configuration - + k_window_test_size = 1000 * 1000 * 8 + 5195 #} END configuration #{ Overrides diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 2027d9004..edc2ec2fe 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,102 +1,11 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import align_to_page -from smmap.mman import Window -from smmap.mman import MappedRegion -from smmap.mman import MappedRegionList -from smmap.mman import Cursor - -import sys -import mmap class TestMMan(TestBase): - _window_test_size = 1000 * 1000 * 8 + 5195 - - def test_window(self): - wl = Window(0, 1) # left - wc = Window(1, 1) # center - wc2 = Window(10, 5) # another center - wr = Window(8000, 50) # right - - assert wl.ofs_end() == 1 - assert wc.ofs_end() == 2 - assert wr.ofs_end() == 8050 - - # extension does nothing if already in place - maxsize = 100 - wc.extend_left_to(wl, maxsize) - assert wc.ofs == 1 and wc.size == 1 - wl.extend_right_to(wc, maxsize) - wl.extend_right_to(wc, maxsize) - assert wl.ofs == 0 and wl.size == 1 - - # an actual left extension - pofs_end = wc2.ofs_end() - wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - - # respects maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - - # without maxsize - wc.extend_right_to(wr, sys.maxint) - assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - - # extend left - wr.extend_left_to(wc2, maxsize) - wr.extend_left_to(wc2, maxsize) - assert wr.size == maxsize - - wr.extend_left_to(wc2, sys.maxint) - assert wr.ofs == wc2.ofs_end() - - wc.align() - assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 - - def test_region(self): - fc = FileCreator(self._window_test_size, "window_test") - half_size = fc.size / 2 - rofs = align_to_page(4200, False) - rfull = MappedRegion(fc.path, 0, fc.size) - rhalfofs = MappedRegion(fc.path, rofs, fc.size) - rhalfsize = MappedRegion(fc.path, 0, half_size) - - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always - - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - - # auto-refcount - assert rfull.client_count() == 1 - rfull2 = rfull - assert rfull.client_count() == 2 - - # window constructor - w = Window.from_region(rfull) - assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - - def test_region_list(self): - fc = FileCreator(100, "sample_file") - ml = MappedRegionList(fc.path) - - assert len(ml) == 0 - assert ml.path() == fc.path - assert ml.file_size() == fc.size - def test_cursor(self): - pass + man = MappedMemoryManager() def test_basics(self): pass diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py new file mode 100644 index 000000000..815391613 --- /dev/null +++ b/smmap/test/test_util.py @@ -0,0 +1,89 @@ +from lib import TestBase, FileCreator + +from smmap.util import * + +import sys + +class TestMMan(TestBase): + + def test_window(self): + wl = MemoryWindow(0, 1) # left + wc = MemoryWindow(1, 1) # center + wc2 = MemoryWindow(10, 5) # another center + wr = MemoryWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == PAGESIZE*2 + + def test_region(self): + fc = FileCreator(self.k_window_test_size, "window_test") + half_size = fc.size / 2 + rofs = align_to_page(4200, False) + rfull = MappedRegion(fc.path, 0, fc.size) + rhalfofs = MappedRegion(fc.path, rofs, fc.size) + rhalfsize = MappedRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + + # window constructor + w = MemoryWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + fc = FileCreator(100, "sample_file") + ml = MappedRegionList(fc.path) + + assert len(ml) == 0 + assert ml.path() == fc.path + assert ml.file_size() == fc.size + diff --git a/smmap/util.py b/smmap/util.py new file mode 100644 index 000000000..784227752 --- /dev/null +++ b/smmap/util.py @@ -0,0 +1,189 @@ +"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +import os +import sys +import mmap + +from mmap import PAGESIZE +from sys import getrefcount + +__all__ = ["align_to_page", "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] + +#{ Utilities + +def align_to_page(num, round_up): + """Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / PAGESIZE) * PAGESIZE; + if round_up and (res != num): + res += PAGESIZE; + #END handle size + return res; + +#}END utilities + + +#{ Utility Classes + +class MemoryWindow(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "MemoryWindow(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region.ofs_begin(), region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + self.ofs = align_to_page(self.ofs, 0) + self.size = align_to_page(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class MappedRegion(object): + """Defines a mapped region of memory, aligned to pagesizes + :note: deallocates used region automatically on destruction""" + __slots__ = [ + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_ms', # actual size of the mapping + '__weakref__' # allow weak references to a region + ] + _need_compat_layer = sys.version_info[1] < 6 + + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + + + def __init__(self, path, ofs, size): + """Initialize a region, allocate the memory map + :param path: path to the file to map + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._uc = 0 + + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + try: + kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + sizeofs = 0 + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) + + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, size) + #END handle buffer wrapping + finally: + os.close(fd) + #END close file handle + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return len(self._mf) + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self.size() + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + + def client_count(self): + """:return: number of clients currently using this region""" + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 + + def adjust_client_count(self, ofs): + """Adjust the client count by the given positive or negative offset""" + self._nc += ofs + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def adjust_usage_count(self, ofs): + """Adjust the usage count by the given positive or negative offset""" + self._uc += ofs + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return len(self._mf) - self._b + + def ofs_end(self): + return len(self._mf) + #END handle compat layer + + +class MappedRegionList(list): + """List of MappedRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path', # path which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MappedRegionList, cls).__new__(cls) + + def __init__(self, path): + self._path = path + self._file_size = None + + def path(self): + """:return: path to file whose regions we manage""" + return self._path + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + self._file_size = os.stat(self._path).st_size + #END update file size + return self._file_size + +#} END utilty classes From 9fe8f93439118c7ea6c4d5dece879b3849f0931e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 23:50:19 +0200 Subject: [PATCH 0147/3719] Implemented first basic functionality of cursor, which is only complete once some more of the memory manager is implemented. Next up is its major use_window method --- smmap/mman.py | 106 +++++++++++++++++++++++++++++++++++++--- smmap/test/test_mman.py | 15 +++++- smmap/test/test_util.py | 5 ++ smmap/util.py | 12 ++--- 4 files changed, 120 insertions(+), 18 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 0ecc1aad1..af414eada 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,29 +1,34 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" - -__all__ = ["MappedMemoryManager", "MemoryCursor"] - from util import ( MemoryWindow, MappedRegion, MappedRegionList, ) +from weakref import proxy + +__all__ = ["MappedMemoryManager"] +#{ Utilities + +#}END utilities class MemoryCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed""" + alive until it is destroyed. + + Cursors should not be created manually, but are instead returned by the MappedMemoryManager""" __slots__ = ( '_manager', # the manger keeping all file regions - '_regions', # a regions list with regions for our file - '_region', # WEAK REF to our current region + '_rlist', # a regions list with regions for our file + '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide ) def __init__(self, manager = None, regions = None): self._manager = manager - self._regions = regions - self._region = region + self._rlist = regions + self._region = None self._ofs = 0 self._size = 0 @@ -32,12 +37,97 @@ def __del__(self): def _destroy(self): """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._files.pop(self._rlist.path()) + #END remove regions list from manager + #END handle regions def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = rhs._rlist + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + if self._region is not None: + self._region.increment_usage_count(1) + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternativly, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset, size): + """Assure we point to a window which allows access to the given offset into the file + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed becaues we reached the end of the file + :note: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + + def unuse_region(self): + """Unuse the ucrrent region. Does nothing if we have no current region + :note: the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + self._region = None + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor""" + return self._region.ofs_begin() + self._ofs + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region_ref(self): + """:return: weak proxy to our mapped region. + :raise AssertionError: if we have no current region. This is only useful for debugging""" + if self._region is None: + raise AssertionError("region not set") + return proxy(self._region) + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + :note: always False if the cursor does not point to a valid region""" + if self._region is None: + return False + return (self.ofs_begin() <= ofs) and (ofs < self.ofs_end()) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path(self): + """:return: path of the underlying mapped file""" + return self._rlist.path() #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index edc2ec2fe..85a0a1bf5 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,11 +1,22 @@ from lib import TestBase, FileCreator +from copy import copy from smmap.mman import * +from smmap.mman import MemoryCursor class TestMMan(TestBase): def test_cursor(self): - man = MappedMemoryManager() + man = MappedMemoryManager() + c = MemoryCursor(man) + assert not c.is_valid() + assert not c.is_associated() - def test_basics(self): + # copy module + + # assign method + + + + def test_memory_manager(self): pass diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 815391613..2a0e0551e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -75,6 +75,11 @@ def test_region(self): rfull2 = rfull assert rfull.client_count() == 2 + # usage + assert rfull.usage_count() == 0 + rfull.increment_usage_count() + assert rfull.usage_count() == 1 + # window constructor w = MemoryWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() diff --git a/smmap/util.py b/smmap/util.py index 784227752..e4c5c82d6 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -75,8 +75,8 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages - '_ms', # actual size of the mapping - '__weakref__' # allow weak references to a region + '_ms' # actual size of the mapping + '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 @@ -139,17 +139,13 @@ def client_count(self): # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 - def adjust_client_count(self, ofs): - """Adjust the client count by the given positive or negative offset""" - self._nc += ofs - def usage_count(self): """:return: amount of usages so far""" return self._uc - def adjust_usage_count(self, ofs): + def increment_usage_count(self): """Adjust the usage count by the given positive or negative offset""" - self._uc += ofs + self._uc += 1 # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: From 226f42858faab545df0e0a9636c84791f0b3a080 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 01:13:52 +0200 Subject: [PATCH 0148/3719] First bit of MappedMemoryManager started. Much more to come - got lost in git++ and improved it in the meanwhile --- smmap/exc.py | 7 +++++++ smmap/mman.py | 22 +++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 smmap/exc.py diff --git a/smmap/exc.py b/smmap/exc.py new file mode 100644 index 000000000..a090d24d5 --- /dev/null +++ b/smmap/exc.py @@ -0,0 +1,7 @@ +"""Module with system exceptions""" + +class MemoryManagerError(Exception): + """Base class for all exceptions thrown by the memory manager""" + +class RegionCollectionError(MemoryManagerError): + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index af414eada..0d8c8dce4 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -5,6 +5,7 @@ MappedRegionList, ) +from exc import RegionCollectionError from weakref import proxy __all__ = ["MappedMemoryManager"] @@ -45,7 +46,7 @@ def _destroy(self): num_clients = self._rlist.client_count() - 2 if num_clients == 0 and len(self._rlist) == 0: # Free all resources associated with the mapped file - self._manager._files.pop(self._rlist.path()) + self._manager._fdict.pop(self._rlist.path()) #END remove regions list from manager #END handle regions @@ -82,6 +83,7 @@ def use_region(self, offset, size): This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region @@ -146,5 +148,19 @@ class MappedMemoryManager(object): a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" - __slots__ = tuple() - + __slots__ = [ + '_fdict', # mapping of path -> MappedRegionList + '_max_window_size', # maximum size of a window + '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_handles', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + def _collect_one_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :raise RegionCollectionError: + :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + From 04b9ec1e2c0bf657bf575a72a27acdbf2935ecf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 11:14:06 +0200 Subject: [PATCH 0149/3719] Fully implemented the manager - now the cursor can be implenented as well --- smmap/mman.py | 102 +++++++++++++++++++++++++++++++++++++++- smmap/test/test_mman.py | 22 ++++++++- smmap/test/test_util.py | 5 ++ smmap/util.py | 7 ++- 4 files changed, 131 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 0d8c8dce4..9c5432bd1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -3,6 +3,8 @@ MemoryWindow, MappedRegion, MappedRegionList, + is_64_bit, + PAGESIZE ) from exc import RegionCollectionError @@ -150,17 +152,113 @@ class MappedMemoryManager(object): __slots__ = [ '_fdict', # mapping of path -> MappedRegionList - '_max_window_size', # maximum size of a window + '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate - '_max_handles', # maximum amount of handles to keep open + '_max_handle_count', # maximum amount of handles to keep open '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles ] + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): + """initialize the manager with the given parameters. + :param window_size: if 0, a default window size will be chosen depending on + the operating system's architechture. It will internally be quantified to a multiple of the page size + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default iwll be set dependning on the system's architecture. + :param max_open_handles: if not ~0, lmit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, + the manager will free as many handles as posisble""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size == 0: + coeff = 32 + if is_64_bit(): + coeff = 1024 + #END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 512 + if is_64_bit(): + coeff = 8192 + #END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + #END handle max memory size + def _collect_one_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :raise RegionCollectionError: :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-1 == 0 and + (lru_region is None or region.usage_count() < lru_region.usage_count())): + lru_region = region + lru_list = regions + # END update lru_region + #END for each region + #END for each regions list + + if lru_region is None: + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + #END while there is more memory to free + + #{ Interface + def make_cursor(self, path): + """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" + regions = self._fdict.get(path) + if regions is None: + regions = MappedRegionList(path) + self._fdict[path] = regions + # END obtain region for path + return MemoryCursor(self, regions) + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + def page_size(self): + """:return: size of a single memory page in bytes""" + return PAGESIZE + + #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 85a0a1bf5..92c201c78 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,8 +1,13 @@ from lib import TestBase, FileCreator -from copy import copy from smmap.mman import * from smmap.mman import MemoryCursor +from smmap.util import PAGESIZE + +from smmap.exc import RegionCollectionError + +import sys +from copy import copy class TestMMan(TestBase): @@ -19,4 +24,17 @@ def test_cursor(self): def test_memory_manager(self): - pass + man = MappedMemoryManager() + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + assert man.window_size() > 0 + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + assert man.page_size() == PAGESIZE + + # collection doesn't raise in 'any' mode + man._collect_one_lru_region(0) + # doesn't raise if we are within the limit + man._collect_one_lru_region(10) + # raises outside of limit + self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 2a0e0551e..9866217d1 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -92,3 +92,8 @@ def test_region_list(self): assert ml.path() == fc.path assert ml.file_size() == fc.size + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_page(1, False) == 0 + assert align_to_page(1, True) == PAGESIZE + diff --git a/smmap/util.py b/smmap/util.py index e4c5c82d6..7f42834c4 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -6,7 +6,8 @@ from mmap import PAGESIZE from sys import getrefcount -__all__ = ["align_to_page", "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] +__all__ = [ "align_to_page", "is_64_bit", + "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] #{ Utilities @@ -20,6 +21,10 @@ def align_to_page(num, round_up): res += PAGESIZE; #END handle size return res; + +def is_64_bit(): + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxint > (1<<32) - 1 #}END utilities From e7b0e1c2c55c52b144671a1f3a8433901e42804a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 12:56:56 +0200 Subject: [PATCH 0150/3719] Implemented use_region. Let the testing begin. Especially the actual data handling will be interesting, which has to work exclusively through buffer objects. There are plenty of layers between the user and the data, which will always be copied when slicing it (as we have no memoryview). The latter one could be implemented on in case we use 2.7 at some point. Now, let the testing begin ! --- smmap/mman.py | 131 ++++++++++++++++++++++++++++++++++++++-- smmap/test/test_mman.py | 38 ++++++++++-- smmap/test/test_util.py | 2 + smmap/util.py | 16 ++++- 4 files changed, 174 insertions(+), 13 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9c5432bd1..1261fc16c 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -9,6 +9,7 @@ from exc import RegionCollectionError from weakref import proxy +import sys __all__ = ["MappedMemoryManager"] #{ Utilities @@ -77,7 +78,7 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size): + def use_region(self, offset, size, _is_recursive=False): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map @@ -85,15 +86,130 @@ def use_region(self, offset, size): This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + size = min(size, man.window_size()) # clamp size to window size + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + if need_region: + # abort on offsets beyond our mapped file's size - currently we are invalid + if offset > self.file_size(): + return self + # END handle offset too large + + existing_region = None + for region in self._rlist: + if region.includes_ofs(offset): + existing_region = region + break + #END handle existing region + #END for each existing region + if existing_region is None: + left = MemoryWindow(0, 0) + mid = MemoryWindow(offset, size) + right = MemoryWindow(self.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + man._collect_lru_region(man.window_size()) + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(self._rlist) + if len_regions == 1: + if self._rlist[0].ofs_begin() <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(self._rlist): + if region.ofs_begin() > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = MemoryWindow.from_region(self._rlist[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = MemoryWindow.from_region(self._rlist[insert_pos]) + # END adjust right window + left = MemoryWindow.from_region(self._rlist[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, man._window_size) + mid.extend_right_to(right, man._window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if man._handle_count >= man._max_handle_count: + raise Exception + #END assert own imposed max file handles + self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if _is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + man._collect_lru_region(0) + return self.use_region(offset, size, True) + #END handle exceptions + + man._handle_count += 1 + man._memory_size += self._region.size() + self._rlist.insert(insert_pos, self._region) + else: + self._region = existing_region + #END need region handling + #END handle acquire region + + self._region.increment_usage_count() + self._ofs = offset - self._region.ofs_begin() + self._size = min(size, self._region.ofs_end() - offset) + + return self + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region :note: the cursor unuses the region automatically upon destruction. It is recommended to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None - + + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested + :note: You can only obtain a buffer if this instance is_valid() !""" + return buffer(self._region.buffer(), self._ofs, self._size) + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None @@ -136,7 +252,6 @@ def path(self): #} END interface - class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. @@ -161,13 +276,13 @@ class MappedMemoryManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. :param window_size: if 0, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. - :param max_open_handles: if not ~0, lmit the amount of open file handles to the given number. + :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, the manager will free as many handles as posisble""" self._fdict = dict() @@ -193,7 +308,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size - def _collect_one_lru_region(self, size): + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region @@ -253,6 +368,10 @@ def mapped_memory_size(self): """:return: amount of bytes currently mapped in total""" return self._memory_size + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 92c201c78..97cd8f62b 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -12,16 +12,36 @@ class TestMMan(TestBase): def test_cursor(self): + fc = FileCreator(self.k_window_test_size, "cursor_test") + man = MappedMemoryManager() - c = MemoryCursor(man) - assert not c.is_valid() - assert not c.is_associated() + ci = MemoryCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated()# but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + # destruction is fine (even multiple times) + cv._destroy() + MemoryCursor(man)._destroy() def test_memory_manager(self): man = MappedMemoryManager() @@ -33,8 +53,14 @@ def test_memory_manager(self): assert man.page_size() == PAGESIZE # collection doesn't raise in 'any' mode - man._collect_one_lru_region(0) + man._collect_lru_region(0) # doesn't raise if we are within the limit - man._collect_one_lru_region(10) + man._collect_lru_region(10) # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint) + self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + c = man.make_cursor(fc.path) + assert c.use_region(10, 10).is_valid() diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 9866217d1..136d99102 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -88,6 +88,8 @@ def test_region_list(self): fc = FileCreator(100, "sample_file") ml = MappedRegionList(fc.path) + assert ml.client_count() == 1 + assert len(ml) == 0 assert ml.path() == fc.path assert ml.file_size() == fc.size diff --git a/smmap/util.py b/smmap/util.py index 7f42834c4..456562a5f 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -54,7 +54,10 @@ def ofs_end(self): return self.ofs + self.size def align(self): - self.ofs = align_to_page(self.ofs, 0) + """Assures the previous window area is contained in the new one""" + nofs = align_to_page(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs self.size = align_to_page(self.size, 1) def extend_left_to(self, window, max_size): @@ -123,6 +126,10 @@ def __init__(self, path, ofs, size): os.close(fd) #END close file handle + def buffer(self): + """:return: a sliceable buffer which can be used to access the mapped memory""" + return self._mf + def ofs_begin(self): """:return: absolute byte offset to the first byte of the mapping""" return self._b @@ -159,6 +166,9 @@ def size(self): def ofs_end(self): return len(self._mf) + + def buffer(self): + return self._mfb #END handle compat layer @@ -176,6 +186,10 @@ def __init__(self, path): self._path = path self._file_size = None + def client_count(self): + """:return: amount of clients which hold a reference to this instance""" + return getrefcount(self)-3 + def path(self): """:return: path to file whose regions we manage""" return self._path From 997a580c0a14cfa0bed11477cd64198199ef99b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 15:29:52 +0200 Subject: [PATCH 0151/3719] implemented plenty of operational testing. It shows that its not yet working properly. Intersting, it was so promising, and went just a little bit too smooth. There we go :) --- smmap/mman.py | 13 ++++-- smmap/test/test_mman.py | 100 ++++++++++++++++++++++++++++++++++++++-- smmap/util.py | 1 - 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 1261fc16c..8a9066cdc 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -8,7 +8,7 @@ ) from exc import RegionCollectionError -from weakref import proxy +from weakref import ref import sys __all__ = ["MappedMemoryManager"] @@ -100,7 +100,7 @@ def use_region(self, offset, size, _is_recursive=False): if need_region: # abort on offsets beyond our mapped file's size - currently we are invalid - if offset > self.file_size(): + if offset >= self.file_size(): return self # END handle offset too large @@ -222,16 +222,21 @@ def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor""" return self._region.ofs_begin() + self._ofs + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region.ofs_begin() + self._ofs + self._size + def size(self): """:return: amount of bytes we point to""" return self._size def region_ref(self): - """:return: weak proxy to our mapped region. + """:return: weak ref to our mapped region. :raise AssertionError: if we have no current region. This is only useful for debugging""" if self._region is None: raise AssertionError("region not set") - return proxy(self._region) + return ref(self._region) def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 97cd8f62b..f592fc129 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -2,10 +2,11 @@ from smmap.mman import * from smmap.mman import MemoryCursor -from smmap.util import PAGESIZE - +from smmap.util import PAGESIZE, align_to_page from smmap.exc import RegionCollectionError +from random import randint +from time import time import sys from copy import copy @@ -59,8 +60,101 @@ def test_memory_manager(self): # raises outside of limit self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) - # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") c = man.make_cursor(fc.path) assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + fc = FileCreator(self.k_window_test_size, "manager_operation_test") + data = open(fc.path, 'rb').read() + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15) + c = man.make_cursor(fc.path) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + size = man.window_size() / 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + assert c.size() == size + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # obtain second window, which spans the first part of the file - it is a still the same window + assert c.use_region(0, size-10).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == size-10 + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:size-10] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - size + overshoot + assert c.use_region(base_offset, size).is_valid() + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 15000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + while num_random_accesses: + num_random_accesses += 1 + base_offset = randint(0, fc.size - 1) + + # precondition + assert man.max_mapped_memory_size() >= man.mapped_memory_size() + assert man.max_file_handles() >= man.num_file_handles() + + assert c.use_region(base_offset, size).is_valid() + assert c.buffer()[:] == data[base_offset:base_offset+size] + memory_read += c.size() + + assert c.includes_ofs(base_offset) + assert c.includes_ofs(base_offset+c.size()-1) + assert not c.includes_ofs(base_offset+c.size()) + # END while we should do an access + elapsed = time() - st + mb = 1000 * 1000 + sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" + % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) + diff --git a/smmap/util.py b/smmap/util.py index 456562a5f..55e007554 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -83,7 +83,6 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages - '_ms' # actual size of the mapping '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 From c90cfef5597b403ae03edf021442f9f44139561f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 15:56:43 +0200 Subject: [PATCH 0152/3719] Fixed a few little issues, the random access test/perftest now work perfectly. Its not too slow either, 160MB/s compared to the 320MB/s that the c++ implementation gets in release mode. Its odd that for some reason, the Debug version of the c++ implementation is at 940MB/s, which is more like the performance i would have expected --- smmap/mman.py | 4 ++-- smmap/test/test_mman.py | 5 ++--- smmap/util.py | 11 +++++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 8a9066cdc..09fa8fcf9 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -244,7 +244,7 @@ def includes_ofs(self, ofs): :note: always False if the cursor does not point to a valid region""" if self._region is None: return False - return (self.ofs_begin() <= ofs) and (ofs < self.ofs_end()) + return self.ofs_begin() <= ofs < self.ofs_end() def file_size(self): """:return: size of the underlying file""" @@ -326,7 +326,7 @@ def _collect_lru_region(self, size): for regions in self._fdict.itervalues(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-1 == 0 and + if (region.client_count()-2 == 0 and (lru_region is None or region.usage_count() < lru_region.usage_count())): lru_region = region lru_list = regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index f592fc129..071615269 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -138,15 +138,14 @@ def test_memman_operation(self): st = time() while num_random_accesses: - num_random_accesses += 1 + num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) # precondition assert man.max_mapped_memory_size() >= man.mapped_memory_size() assert man.max_file_handles() >= man.num_file_handles() - assert c.use_region(base_offset, size).is_valid() - assert c.buffer()[:] == data[base_offset:base_offset+size] + assert c.buffer()[:] == data[base_offset:base_offset+c.size()] memory_read += c.size() assert c.includes_ofs(base_offset) diff --git a/smmap/util.py b/smmap/util.py index 55e007554..1aae4f534 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -116,7 +116,7 @@ def __init__(self, path, ofs, size): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) @@ -125,6 +125,11 @@ def __init__(self, path, ofs, size): os.close(fd) #END close file handle + def __repr__(self): + return "MappedRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface + def buffer(self): """:return: a sliceable buffer which can be used to access the mapped memory""" return self._mf @@ -143,7 +148,7 @@ def ofs_end(self): def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" - return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + return self.ofs_begin() <= ofs < self.ofs_end() def client_count(self): """:return: number of clients currently using this region""" @@ -170,6 +175,8 @@ def buffer(self): return self._mfb #END handle compat layer + #} END interface + class MappedRegionList(list): """List of MappedRegion instances associating a path with a list of regions.""" From 4f3d1ad43740fde43a321abcf59676f8ab9c693f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 16:52:15 +0200 Subject: [PATCH 0153/3719] Applied some optimizations for performance. Python is very easily overwhelmed with plenty of calls, causing too much overhead --- smmap/mman.py | 66 +++++++++++++++++++++++++---------------- smmap/test/test_mman.py | 28 ++++++++++------- smmap/util.py | 19 +++++++----- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 09fa8fcf9..c13ef6599 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -99,18 +99,30 @@ def use_region(self, offset, size, _is_recursive=False): # END check existing region if need_region: + window_size = man._window_size + # abort on offsets beyond our mapped file's size - currently we are invalid if offset >= self.file_size(): return self # END handle offset too large existing_region = None - for region in self._rlist: - if region.includes_ofs(offset): - existing_region = region - break - #END handle existing region - #END for each existing region + a = self._rlist + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + existing_region = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting if existing_region is None: left = MemoryWindow(0, 0) @@ -119,20 +131,23 @@ def use_region(self, offset, size, _is_recursive=False): # we want to honor the max memory size, and assure we have anough # memory available - man._collect_lru_region(man.window_size()) + # Save calls ! + if self._manager._memory_size + window_size > self._manager._max_memory_size: + man._collect_lru_region(window_size) + #END handle collection # we assume the list remains sorted by offset insert_pos = 0 - len_regions = len(self._rlist) + len_regions = len(a) if len_regions == 1: - if self._rlist[0].ofs_begin() <= offset: + if a[0]._b <= offset: insert_pos = 1 #END maintain sort else: # find insert position insert_pos = len_regions - for i, region in enumerate(self._rlist): - if region.ofs_begin() > offset: + for i, region in enumerate(a): + if region._b > offset: insert_pos = i break #END if insert position is correct @@ -143,17 +158,17 @@ def use_region(self, offset, size, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = MemoryWindow.from_region(self._rlist[insert_pos]) + right = MemoryWindow.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = MemoryWindow.from_region(self._rlist[insert_pos]) + right = MemoryWindow.from_region(a[insert_pos]) # END adjust right window - left = MemoryWindow.from_region(self._rlist[insert_pos - 1]) + left = MemoryWindow.from_region(a[insert_pos - 1]) #END adjust surrounding windows - mid.extend_left_to(left, man._window_size) - mid.extend_right_to(right, man._window_size) + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) mid.align() # it can happen that we align beyond the end of the file @@ -166,7 +181,7 @@ def use_region(self, offset, size, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size) + self._region = MappedRegion(a.path(), mid.ofs, mid.size) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -185,14 +200,14 @@ def use_region(self, offset, size, _is_recursive=False): man._handle_count += 1 man._memory_size += self._region.size() - self._rlist.insert(insert_pos, self._region) + a.insert(insert_pos, self._region) else: self._region = existing_region #END need region handling #END handle acquire region self._region.increment_usage_count() - self._ofs = offset - self._region.ofs_begin() + self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) return self @@ -220,12 +235,12 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor""" - return self._region.ofs_begin() + self._ofs + return self._region._b + self._ofs def ofs_end(self): """:return: offset to one past the last available byte""" # unroll method calls for performance ! - return self._region.ofs_begin() + self._ofs + self._size + return self._region._b + self._ofs + self._size def size(self): """:return: amount of bytes we point to""" @@ -241,10 +256,9 @@ def region_ref(self): def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - :note: always False if the cursor does not point to a valid region""" - if self._region is None: - return False - return self.ofs_begin() <= ofs < self.ofs_end() + :note: cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) def file_size(self): """:return: size of the underlying file""" @@ -327,7 +341,7 @@ def _collect_lru_region(self, size): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and - (lru_region is None or region.usage_count() < lru_region.usage_count())): + (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 071615269..6f3809979 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -127,9 +127,6 @@ def test_memman_operation(self): # remove mapped regions if we have to assert man.num_file_handles() == 2 - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - # iterate through the windows, verify data contents # this will trigger map collection after a while max_random_accesses = 15000 @@ -137,23 +134,32 @@ def test_memman_operation(self): memory_read = 0 st = time() + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles while num_random_accesses: num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) # precondition - assert man.max_mapped_memory_size() >= man.mapped_memory_size() - assert man.max_file_handles() >= man.num_file_handles() + assert max_mapped_memory_size >= mapped_memory_size() + assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, size).is_valid() - assert c.buffer()[:] == data[base_offset:base_offset+c.size()] - memory_read += c.size() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize - assert c.includes_ofs(base_offset) - assert c.includes_ofs(base_offset+c.size()-1) - assert not c.includes_ofs(base_offset+c.size()) + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = time() - st mb = 1000 * 1000 - sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" + sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() diff --git a/smmap/util.py b/smmap/util.py index 1aae4f534..6a2056b5d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,7 +3,7 @@ import sys import mmap -from mmap import PAGESIZE +from mmap import PAGESIZE, mmap, ACCESS_READ from sys import getrefcount __all__ = [ "align_to_page", "is_64_bit", @@ -48,7 +48,7 @@ def __repr__(self): @classmethod def from_region(cls, region): """:return: new window from a region""" - return cls(region.ofs_begin(), region.size()) + return cls(region._b, region._size) def ofs_end(self): return self.ofs + self.size @@ -83,6 +83,7 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages + '_size', # cached size of our memory map '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 @@ -100,11 +101,12 @@ def __init__(self, path, ofs, size): allocated the the size automatically adjusted :raise Exception: if no memory can be allocated""" self._b = ofs + self._size = 0 self._uc = 0 fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: - kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size sizeofs = ofs if self._need_compat_layer: @@ -116,7 +118,8 @@ def __init__(self, path, ofs, size): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + self._mf = mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + self._size = len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) @@ -126,7 +129,7 @@ def __init__(self, path, ofs, size): #END close file handle def __repr__(self): - return "MappedRegion<%i, %i>" % (self._b, self.size()) + return "MappedRegion<%i, %i>" % (self._b, self._size) #{ Interface @@ -140,15 +143,15 @@ def ofs_begin(self): def size(self): """:return: total size of the mapped region in bytes""" - return len(self._mf) + return self._size def ofs_end(self): """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self.size() + return self._b + self._size def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" - return self.ofs_begin() <= ofs < self.ofs_end() + return self._b <= ofs < self._b + self._size def client_count(self): """:return: number of clients currently using this region""" From 011343f6f43ea868c3efbd00a43757eb82196e1d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 17:46:24 +0200 Subject: [PATCH 0154/3719] Cursor can now take additional flags when opening the file handle for mapping. Renamed stream to buf, as the first item will be a buffer which uses the cursor underneath. We should add a stream for good measure though --- smmap/{stream.py => buf.py} | 0 smmap/mman.py | 10 +++++++--- smmap/test/{test_stream.py => test_buf.py} | 4 ++-- smmap/util.py | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) rename smmap/{stream.py => buf.py} (100%) rename smmap/test/{test_stream.py => test_buf.py} (54%) diff --git a/smmap/stream.py b/smmap/buf.py similarity index 100% rename from smmap/stream.py rename to smmap/buf.py diff --git a/smmap/mman.py b/smmap/mman.py index c13ef6599..57e2d81b5 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -78,10 +78,12 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, _is_recursive=False): + def use_region(self, offset, size, flags = 0, _is_recursive=False): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, @@ -106,6 +108,8 @@ def use_region(self, offset, size, _is_recursive=False): return self # END handle offset too large + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. existing_region = None a = self._rlist lo = 0 @@ -181,7 +185,7 @@ def use_region(self, offset, size, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(a.path(), mid.ofs, mid.size) + self._region = MappedRegion(a.path(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -195,7 +199,7 @@ def use_region(self, offset, size, _is_recursive=False): raise #END handle existing recursion man._collect_lru_region(0) - return self.use_region(offset, size, True) + return self.use_region(offset, size, flags, True) #END handle exceptions man._handle_count += 1 diff --git a/smmap/test/test_stream.py b/smmap/test/test_buf.py similarity index 54% rename from smmap/test/test_stream.py rename to smmap/test/test_buf.py index fae928d9a..231abcf5b 100644 --- a/smmap/test/test_stream.py +++ b/smmap/test/test_buf.py @@ -1,7 +1,7 @@ from lib import TestBase -from smmap.stream import * +from smmap.buf import * -class TestStream(TestBase): +class TestBuf(TestBase): def test_basics(self): assert False diff --git a/smmap/util.py b/smmap/util.py index 6c5282cb9..786622d9a 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -93,18 +93,19 @@ class MappedRegion(object): #END handle additional slot - def __init__(self, path, ofs, size): + def __init__(self, path, ofs, size, flags = 0): """Initialize a region, allocate the memory map :param path: path to the file to map :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. :raise Exception: if no memory can be allocated""" self._b = ofs self._size = 0 self._uc = 0 - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size From 8670443c3919f5e71e5864717d93eeb7921432ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 22:01:10 +0200 Subject: [PATCH 0155/3719] Implemented buffer interface and test to fully proove it. Its not yet fully properly implemented --- smmap/buf.py | 91 +++++++++++++++++++++++++++++++++++++++-- smmap/mman.py | 12 +++++- smmap/test/test_buf.py | 89 +++++++++++++++++++++++++++++++++++++++- smmap/test/test_mman.py | 14 +++++-- 4 files changed, 196 insertions(+), 10 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index bc5e568ea..f7db963ed 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,7 +1,92 @@ -"""Module with a simple stream implementation using the memory manager""" +"""Module with a simple buffer implementation using the memory manager""" +from mman import MemoryCursor -from mman import * +import sys -__all__ = [] +__all__ = ["MappedMemoryBuffer"] + +class MappedMemoryBuffer(object): + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by an underlying memory manager. + + A buffer, once initialized, stays put on providing access to eactly one path. + A custom interface allows you to change paths mid way, and to optimize + the resource usage. + + Please note that this type is only fully usable if you configure it with the + MappedMemoryManager to use. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at your given offset.""" + __slots__ = '_c' # our cursor + + #{ Configuration + # A subclass must provide an instance of a (usually global) MappedMemoryManager + manager = None + #}END configuration + + def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given path if given. + :param path: if not None, the path to the file you want to access + If None, you have call begin_access before using the buffer + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = MemoryCursor(self.manager) + assert self.manager is not None, "Require the cls.manager variable to be set in subclass" + if path and not self.begin_access(path, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset + + def __del__(self): + self.end_access() + + def __getitem__(self, i): + c = self._c + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + assert c.is_valid() # TODO: remove for performance + return c.buffer()[i] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + if c.ofs_begin() >= i and j < c.ofs_end(): + return c.buffer()[i:j] + raise NotImplementedError() + #{ Interface + + def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if path is empty or None the existing path will be used if possible. + :return: True if the buffer can be used""" + if path and (not self._c.is_associated() or self._c.path() != path): + self._c = self.manager.make_cursor(path) + #END get associated cursor + + # reuse existing cursors if possible + if self._c.is_associated(): + return self._c.use_region(offset, size, flags).is_valid() + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._c.unuse_region() + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/mman.py b/smmap/mman.py index 57e2d81b5..0fa9ef450 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -226,7 +226,9 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested - :note: You can only obtain a buffer if this instance is_valid() !""" + :note: You can only obtain a buffer if this instance is_valid() ! + :note: buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) def is_valid(self): @@ -336,6 +338,7 @@ def _collect_lru_region(self, size): :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :raise RegionCollectionError: + :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -365,6 +368,8 @@ def _collect_lru_region(self, size): self._handle_count -= 1 #END while there is more memory to free + return num_found + #{ Interface def make_cursor(self, path): """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" @@ -375,6 +380,11 @@ def make_cursor(self, path): # END obtain region for path return MemoryCursor(self, regions) + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + def num_file_handles(self): """:return: amount of file handles in use. Each mapped region uses one file handle""" return self._handle_count diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 231abcf5b..a26c8bb09 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,7 +1,92 @@ -from lib import TestBase +from lib import TestBase, FileCreator +from smmap.mman import MappedMemoryManager from smmap.buf import * +from random import randint +from time import time +import sys + +class TestBuffer(MappedMemoryBuffer): + #{ Configuration + manager = MappedMemoryManager() + #} END configuration + + class TestBuf(TestBase): + def test_basics(self): - assert False + self.failUnlessRaises(AssertionError, MappedMemoryBuffer) # needs subclass + fc = FileCreator(self.k_window_test_size, "buffer_test") + + # invalid paths fail upon construction + self.failUnlessRaises(OSError, TestBuffer, "somefile") # invalid file + self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size) # offset too large + + buf = TestBuffer() # can create uninitailized buffers + assert not buf.cursor().is_valid() and not buf.cursor().is_associated() + + # can call end access any time + buf.end_access() + buf.end_access() + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(fc.path, fc.size) == False + assert buf.begin_access(fc.path, offset) == True + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert buf.cursor().is_valid() + + # simple access + data = open(fc.path, 'rb').read() + assert data[offset] == buf[0] + assert data[offset:offset*2] == buf[0:offset] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + + man = TestBuffer.manager + assert man.num_file_handles() == 1 + + # PERFORMANCE + # blast away with rnadom access and a full mapping - we don't want to + # exagerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 5000 + num_accesses_left = max_num_accesses + + for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man): + TestBuffer.manager = manager + st = time() + buf = TestBuffer(fc.path) + assert manager.num_file_handles() == 1 + num_bytes = 0 + fsize = fc.size + while num_accesses_left: + num_accesses_left -= 1 + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + pos = randint(0, fsize) + assert buf[pos] == data[pos] + # END handle num accesses + buf.end_access() + assert manager.num_file_handles() == 1 + assert manager.collect() == 1 + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = 1000*1000 + sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 6f3809979..46a0f50fb 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -75,7 +75,8 @@ def test_memman_operation(self): assert len(data) == fc.size # small windows, a reasonable max memory. Not too many regions at once - man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15) + max_num_handles = 15 + man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) c = man.make_cursor(fc.path) # still empty (more about that is tested in test_memory_manager() @@ -129,7 +130,7 @@ def test_memman_operation(self): # iterate through the windows, verify data contents # this will trigger map collection after a while - max_random_accesses = 15000 + max_random_accesses = 5000 num_random_accesses = max_random_accesses memory_read = 0 st = time() @@ -160,6 +161,11 @@ def test_memman_operation(self): mb = 1000 * 1000 sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) - + # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 From b00bc5e4c0e8eb47d0bf59c0a2c1f5399f4c8f58 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:15:42 +0200 Subject: [PATCH 0156/3719] Finished implementation of buffer including a test which shows the performance should be usable in the real world. Its actually not too bad --- smmap/buf.py | 31 +++++++++++++++++---- smmap/mman.py | 5 +++- smmap/test/test_buf.py | 60 ++++++++++++++++++++++++----------------- smmap/test/test_mman.py | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index f7db963ed..68e7c33b9 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -44,19 +44,40 @@ def __del__(self): def __getitem__(self, i): c = self._c + assert c.is_valid() if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage - assert c.is_valid() # TODO: remove for performance - return c.buffer()[i] + return c.buffer()[i-c.ofs_begin()] def __getslice__(self, i, j): c = self._c # fast path, slice fully included - safes a concatenate operation and # should be the default - if c.ofs_begin() >= i and j < c.ofs_end(): - return c.buffer()[i:j] - raise NotImplementedError() + assert c.is_valid() + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i-b:j-b] + else: + l = j-i # total length + ofs = i + # keep tokens, and join afterwards. This is faster + # as it can preallocate the total amoint of space needed + # (and its verified the implementation does that) + # Question is whether the list allocation doesn't counteract this, + # but lets see ... + tokens = list() + tappend = tokens.append + + while l: + c.use_region(ofs, l) + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + tappend(d) + #END while there are bytes to read + return ''.join(tokens) + # END fast or slow path #{ Interface def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): diff --git a/smmap/mman.py b/smmap/mman.py index 0fa9ef450..588a63563 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -222,6 +222,8 @@ def unuse_region(self): to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! def buffer(self): """Return a buffer object which allows access to our memory region from our offset @@ -240,7 +242,8 @@ def is_associated(self): return self._rlist is not None def ofs_begin(self): - """:return: offset to the first byte pointed to by our cursor""" + """:return: offset to the first byte pointed to by our cursor + :note: only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index a26c8bb09..d7e6d6c96 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -61,32 +61,42 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 5000 - num_accesses_left = max_num_accesses - - for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man): + max_num_accesses = 1000 + for manager, man_id in ( (man, 'optimal'), + (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')): TestBuffer.manager = manager - st = time() buf = TestBuffer(fc.path) assert manager.num_file_handles() == 1 - num_bytes = 0 - fsize = fc.size - while num_accesses_left: - num_accesses_left -= 1 - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - pos = randint(0, fsize) - assert buf[pos] == data[pos] - # END handle num accesses - buf.end_access() - assert manager.num_file_handles() == 1 - assert manager.collect() == 1 - assert manager.num_file_handles() == 0 - elapsed = time() - st - mb = 1000*1000 - sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 46a0f50fb..b1c8f68eb 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -158,7 +158,7 @@ def test_memman_operation(self): assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = time() - st - mb = 1000 * 1000 + mb = float(1000 * 1000) sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) From 68fba826cdfd0de532ccf14c9ad94bf035a36e1b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:22:46 +0200 Subject: [PATCH 0157/3719] Optimized __getslice__ implementation a bit --- smmap/buf.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 68e7c33b9..d1be1b65a 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -61,22 +61,17 @@ def __getslice__(self, i, j): else: l = j-i # total length ofs = i - # keep tokens, and join afterwards. This is faster - # as it can preallocate the total amoint of space needed - # (and its verified the implementation does that) - # Question is whether the list allocation doesn't counteract this, - # but lets see ... - tokens = list() - tappend = tokens.append - + # Keeping tokens in a list could possible be faster, but the list + # overhead outweighs the benefits (tested) ! + md = str() while l: c.use_region(ofs, l) d = c.buffer()[:l] ofs += len(d) l -= len(d) - tappend(d) + md += d #END while there are bytes to read - return ''.join(tokens) + return md # END fast or slow path #{ Interface From c30e930bf06edc8f60ce3311d13f6a61de8ac0ce Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:57:55 +0200 Subject: [PATCH 0158/3719] Changed buffer implementation to use a cursor right away instead of taking a path and a manager. This makes it much more flexible, as it doesn't have to care about the manager anymore, making it easier to use and making clear that it is meant for use with a mapped memory manager implementation. --- smmap/buf.py | 44 ++++++++++++++++-------------------------- smmap/test/test_buf.py | 36 +++++++++++++++++----------------- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index d1be1b65a..b4f581316 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -7,35 +7,23 @@ class MappedMemoryBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into - memory of a mapped file. The mapping is controlled by an underlying memory manager. - - A buffer, once initialized, stays put on providing access to eactly one path. - A custom interface allows you to change paths mid way, and to optimize - the resource usage. - - Please note that this type is only fully usable if you configure it with the - MappedMemoryManager to use. + memory of a mapped file. The mapping is controlled by the provided cursor. The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at your given offset.""" + first byte at the offset you used during initialization or begin_access""" __slots__ = '_c' # our cursor - #{ Configuration - # A subclass must provide an instance of a (usually global) MappedMemoryManager - manager = None - #}END configuration - def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0): - """Initalize the instance to operate on the given path if given. - :param path: if not None, the path to the file you want to access - If None, you have call begin_access before using the buffer + def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size :param flags: Additional flags to be passed to os.open :raise ValueError: if the buffer could not achieve a valid state""" - self._c = MemoryCursor(self.manager) - assert self.manager is not None, "Require the cls.manager variable to be set in subclass" - if path and not self.begin_access(path, offset, size, flags): + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") # END handle offset @@ -75,19 +63,19 @@ def __getslice__(self, i, j): # END fast or slow path #{ Interface - def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): + def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. For more information no the parameters, see the __init__ method - :param path: if path is empty or None the existing path will be used if possible. + :param path: if cursor is None the existing one will be used. :return: True if the buffer can be used""" - if path and (not self._c.is_associated() or self._c.path() != path): - self._c = self.manager.make_cursor(path) - #END get associated cursor + if cursor: + self._c = cursor + #END update our cursor # reuse existing cursors if possible - if self._c.is_associated(): + if self._c is not None and self._c.is_associated(): return self._c.use_region(offset, size, flags).is_valid() return False @@ -97,7 +85,9 @@ def end_access(self): resources to be freed. Once you called end_access, you must call begin access before reusing this instance!""" - self._c.unuse_region() + if self._c is not None: + self._c.unuse_region() + #END unuse region def cursor(self): """:return: the currently set cursor which provides access to the data""" diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d7e6d6c96..6192897bd 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -7,24 +7,24 @@ from time import time import sys -class TestBuffer(MappedMemoryBuffer): - #{ Configuration - manager = MappedMemoryManager() - #} END configuration - + +man_optimal = MappedMemoryManager() +man_worst_case = MappedMemoryManager( window_size=TestBase.k_window_test_size/100, + max_memory_size=TestBase.k_window_test_size/3, + max_open_handles=15) class TestBuf(TestBase): def test_basics(self): - self.failUnlessRaises(AssertionError, MappedMemoryBuffer) # needs subclass fc = FileCreator(self.k_window_test_size, "buffer_test") # invalid paths fail upon construction - self.failUnlessRaises(OSError, TestBuffer, "somefile") # invalid file - self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size) # offset too large + c = man_optimal.make_cursor(fc.path) + self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size) # offset too large - buf = TestBuffer() # can create uninitailized buffers - assert not buf.cursor().is_valid() and not buf.cursor().is_associated() + buf = MappedMemoryBuffer() # can create uninitailized buffers + assert buf.cursor() is None # can call end access any time buf.end_access() @@ -32,8 +32,9 @@ def test_basics(self): # begin access can revive it, if the offset is suitable offset = 100 - assert buf.begin_access(fc.path, fc.size) == False - assert buf.begin_access(fc.path, offset) == True + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True @@ -52,9 +53,9 @@ def test_basics(self): # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically + del(c) - man = TestBuffer.manager - assert man.num_file_handles() == 1 + assert man_optimal.num_file_handles() == 1 # PERFORMANCE # blast away with rnadom access and a full mapping - we don't want to @@ -62,10 +63,9 @@ def test_basics(self): # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 1000 - for manager, man_id in ( (man, 'optimal'), - (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')): - TestBuffer.manager = manager - buf = TestBuffer(fc.path) + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case')): + buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses From aafc980e9cbb1b2ed08d374300e9916538136cc7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 00:02:00 +0200 Subject: [PATCH 0159/3719] Added indirection level to internally used types to allow others to exchange them with their own implementations. --- smmap/mman.py | 25 +++++++++++++++++-------- smmap/test/test_buf.py | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 588a63563..d8626d065 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -29,6 +29,11 @@ class MemoryCursor(object): '_size' # maximum size we should provide ) + #{ Configuration + MemoryWindowCls = MemoryWindow + MappedRegionCls = MappedRegion + #} END configuration + def __init__(self, manager = None, regions = None): self._manager = manager self._rlist = regions @@ -129,9 +134,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): #END while bisecting if existing_region is None: - left = MemoryWindow(0, 0) - mid = MemoryWindow(offset, size) - right = MemoryWindow(self.file_size(), 0) + left = self.MemoryWindowCls(0, 0) + mid = self.MemoryWindowCls(offset, size) + right = self.MemoryWindowCls(self.file_size(), 0) # we want to honor the max memory size, and assure we have anough # memory available @@ -162,13 +167,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = MemoryWindow.from_region(a[insert_pos]) + right = self.MemoryWindowCls.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = MemoryWindow.from_region(a[insert_pos]) + right = self.MemoryWindowCls.from_region(a[insert_pos]) # END adjust right window - left = MemoryWindow.from_region(a[insert_pos - 1]) + left = self.MemoryWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows mid.extend_left_to(left, window_size) @@ -185,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(a.path(), mid.ofs, mid.size, flags) + self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -302,6 +307,10 @@ class MappedMemoryManager(object): '_handle_count', # amount of currently allocated file handles ] + #{ Configuration + MappedRegionListCls = MappedRegionList + #} END configuration + _MB_in_bytes = 1024 * 1024 def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): @@ -378,7 +387,7 @@ def make_cursor(self, path): """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" regions = self._fdict.get(path) if regions is None: - regions = MappedRegionList(path) + regions = self.MappedRegionListCls(path) self._fdict[path] = regions # END obtain region for path return MemoryCursor(self, regions) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 6192897bd..ae1a174d5 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -62,7 +62,7 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 1000 + max_num_accesses = 400 for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case')): buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) From 8d64e74ed80f2818acad652a69615708f8f61104 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 00:38:48 +0200 Subject: [PATCH 0160/3719] Added very special purpose method to free memory maps. The whole reason for this is to make the windows test work after all \! --- smmap/mman.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/smmap/mman.py b/smmap/mman.py index d8626d065..44d985e28 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -426,3 +426,32 @@ def page_size(self): return PAGESIZE #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + :note: does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface From 4ad22313d8764bcc3645ba8d8ea3ff6f50bba7d1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 09:42:55 +0200 Subject: [PATCH 0161/3719] System can now deal with file descriptors as input, but it still requires some more testing. Also fds need to remain open to be usable for new mapped regions --- smmap/mman.py | 41 ++++++++++++++++++++++++++++++++--------- smmap/test/test_util.py | 18 ++++++++++++------ smmap/util.py | 29 +++++++++++++++++++---------- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 44d985e28..79f2de896 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -54,7 +54,7 @@ def _destroy(self): num_clients = self._rlist.client_count() - 2 if num_clients == 0 and len(self._rlist) == 0: # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path()) + self._manager._fdict.pop(self._rlist.path_or_fd()) #END remove regions list from manager #END handle regions @@ -190,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags) + self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -278,9 +278,26 @@ def file_size(self): """:return: size of the underlying file""" return self._rlist.file_size() + def path_or_fd(self): + """:return: path or file decriptor of the underlying mapped file""" + return self._rlist.path_or_fd() + def path(self): - """:return: path of the underlying mapped file""" - return self._rlist.path() + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + :note: it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), basestring): + return ValueError("File descriptor queried although mapping was generated from path") + #END handle type + return self._rlist.path_or_fd() #} END interface @@ -383,12 +400,18 @@ def _collect_lru_region(self, size): return num_found #{ Interface - def make_cursor(self, path): - """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" - regions = self._fdict.get(path) + def make_cursor(self, path_or_fd): + """:return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + :note: if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open.""" + regions = self._fdict.get(path_or_fd) if regions is None: - regions = self.MappedRegionListCls(path) - self._fdict[path] = regions + regions = self.MappedRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions # END obtain region for path return MemoryCursor(self, regions) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 136d99102..a5478cd7c 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -2,6 +2,7 @@ from smmap.util import * +import os import sys class TestMMan(TestBase): @@ -86,13 +87,18 @@ def test_region(self): def test_region_list(self): fc = FileCreator(100, "sample_file") - ml = MappedRegionList(fc.path) - assert ml.client_count() == 1 - - assert len(ml) == 0 - assert ml.path() == fc.path - assert ml.file_size() == fc.size + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + ml = MappedRegionList(item) + + assert ml.client_count() == 1 + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + #END handle input + os.close(fd) def test_util(self): assert isinstance(is_64_bit(), bool) # just call it diff --git a/smmap/util.py b/smmap/util.py index 786622d9a..f3bb58b33 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -93,9 +93,9 @@ class MappedRegion(object): #END handle additional slot - def __init__(self, path, ofs, size, flags = 0): + def __init__(self, path_or_fd, ofs, size, flags = 0): """Initialize a region, allocate the memory map - :param path: path to the file to map + :param path_or_fd: path to the file to map, or the opened file descriptor :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted @@ -105,7 +105,12 @@ def __init__(self, path, ofs, size, flags = 0): self._size = 0 self._uc = 0 - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + #END handle fd + try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size @@ -189,29 +194,33 @@ def includes_ofs(self, ofs): class MappedRegionList(list): """List of MappedRegion instances associating a path with a list of regions.""" __slots__ = ( - '_path', # path which is mapped by all our regions + '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): return super(MappedRegionList, cls).__new__(cls) - def __init__(self, path): - self._path = path + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd self._file_size = None def client_count(self): """:return: amount of clients which hold a reference to this instance""" return getrefcount(self)-3 - def path(self): - """:return: path to file whose regions we manage""" - return self._path + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd def file_size(self): """:return: size of file we manager""" if self._file_size is None: - self._file_size = os.stat(self._path).st_size + if isinstance(self._path_or_fd, basestring): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + #END handle path type #END update file size return self._file_size From a8a5e10835d71bb99993744777a06fc5573c892c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 09:53:10 +0200 Subject: [PATCH 0162/3719] Added tests for the fd case. It shows that this is measurably faster than the string path version because there is less system overhead. Good to know actally --- smmap/mman.py | 4 +- smmap/test/test_buf.py | 80 +++++++++------- smmap/test/test_mman.py | 207 +++++++++++++++++++++------------------- smmap/util.py | 4 +- 4 files changed, 157 insertions(+), 138 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 79f2de896..b15b9af7c 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -407,7 +407,9 @@ def make_cursor(self, path_or_fd): but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open.""" + are preferred unless you plan to keep the file descriptor open. + :note: Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: regions = self.MappedRegionListCls(path_or_fd) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index ae1a174d5..efc1da60c 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -6,6 +6,7 @@ from random import randint from time import time import sys +import os man_optimal = MappedMemoryManager() @@ -63,40 +64,45 @@ def test_basics(self): # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 400 - for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case')): - buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - #END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = time() - st - mb = float(1000*1000) - mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) - # END handle access mode - # END for each manager + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case')): + buf = MappedMemoryBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index b1c8f68eb..57d78d504 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -7,6 +7,7 @@ from random import randint from time import time +import os import sys from copy import copy @@ -62,110 +63,118 @@ def test_memory_manager(self): # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") - c = man.make_cursor(fc.path) - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + #END for each input + os.close(fd) def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") data = open(fc.path, 'rb').read() - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - max_num_handles = 15 - man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) - c = man.make_cursor(fc.path) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - assert c.size() == size - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) - - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # obtain second window, which spans the first part of the file - it is a still the same window - assert c.use_region(0, size-10).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == size-10 - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:size-10] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - size + overshoot - assert c.use_region(base_offset, size).is_valid() - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + max_num_handles = 15 + man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) + c = man.make_cursor(item) - # precondition - assert max_mapped_memory_size >= mapped_memory_size() - assert max_file_handles >= num_file_handles() + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + size = man.window_size() / 2 assert c.use_region(base_offset, size).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = time() - st - mb = float(1000 * 1000) - sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" - % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + assert c.size() == size + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # obtain second window, which spans the first part of the file - it is a still the same window + assert c.use_region(0, size-10).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == size-10 + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:size-10] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - size + overshoot + assert c.use_region(base_offset, size).is_valid() + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + assert max_mapped_memory_size >= mapped_memory_size() + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, size).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = time() - st + mb = float(1000 * 1000) + sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + os.close(fd) diff --git a/smmap/util.py b/smmap/util.py index f3bb58b33..2ba7a1d29 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -131,7 +131,9 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._mfb = buffer(self._mf, ofs, size) #END handle buffer wrapping finally: - os.close(fd) + if isinstance(path_or_fd, basestring): + os.close(fd) + #END only close it if we opened it #END close file handle def __repr__(self): From 9f16040fb4cde025875e28b01d70cfba11c1d773 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 10:42:32 +0200 Subject: [PATCH 0163/3719] Fixed some mapping issues on windows. Fixed some tests to deal with the very different granularity --- smmap/mman.py | 5 ----- smmap/test/test_buf.py | 2 +- smmap/test/test_mman.py | 7 +++---- smmap/test/test_util.py | 17 ++++++++++++----- smmap/util.py | 16 ++++++++-------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index b15b9af7c..8e50a1445 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MappedRegion, MappedRegionList, is_64_bit, - PAGESIZE ) from exc import RegionCollectionError @@ -446,10 +445,6 @@ def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size - def page_size(self): - """:return: size of a single memory page in bytes""" - return PAGESIZE - #} END interface #{ Special Purpose Interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index efc1da60c..c772e4999 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -97,7 +97,7 @@ def test_basics(self): assert manager.num_file_handles() assert manager.collect() assert manager.num_file_handles() == 0 - elapsed = time() - st + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 57d78d504..e220f8bb2 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -2,7 +2,7 @@ from smmap.mman import * from smmap.mman import MemoryCursor -from smmap.util import PAGESIZE, align_to_page +from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError from random import randint @@ -52,7 +52,6 @@ def test_memory_manager(self): assert man.window_size() > 0 assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 - assert man.page_size() == PAGESIZE # collection doesn't raise in 'any' mode man._collect_lru_region(0) @@ -102,7 +101,7 @@ def test_memman_operation(self): assert c.size() == size assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) assert c.buffer()[:] == data[base_offset:base_offset+size] @@ -164,7 +163,7 @@ def test_memman_operation(self): assert includes_ofs(base_offset+csize-1) assert not includes_ofs(base_offset+csize) # END while we should do an access - elapsed = time() - st + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index a5478cd7c..7caa427ba 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,6 +1,7 @@ from lib import TestBase, FileCreator from smmap.util import * +from mmap import ALLOCATIONGRANULARITY import os import sys @@ -50,12 +51,12 @@ def test_window(self): assert wr.ofs == wc2.ofs_end() wc.align() - assert wc.ofs == 0 and wc.size == PAGESIZE*2 + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size / 2 - rofs = align_to_page(4200, False) + rofs = align_to_mmap(4200, False) rfull = MappedRegion(fc.path, 0, fc.size) rhalfofs = MappedRegion(fc.path, rofs, fc.size) rhalfsize = MappedRegion(fc.path, 0, half_size) @@ -69,7 +70,13 @@ def test_region(self): assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + # with the values we have, this test only works on windows where an alignment + # size of 4096 is assumed. + if sys.platform == 'win32': + assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0) + else: + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + #END handle platforms # auto-refcount assert rfull.client_count() == 1 @@ -102,6 +109,6 @@ def test_region_list(self): def test_util(self): assert isinstance(is_64_bit(), bool) # just call it - assert align_to_page(1, False) == 0 - assert align_to_page(1, True) == PAGESIZE + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY diff --git a/smmap/util.py b/smmap/util.py index 2ba7a1d29..e1f786538 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,22 +3,22 @@ import sys import mmap -from mmap import PAGESIZE, mmap, ACCESS_READ +from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ from sys import getrefcount -__all__ = [ "align_to_page", "is_64_bit", - "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] +__all__ = [ "align_to_mmap", "is_64_bit", + "MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities -def align_to_page(num, round_up): +def align_to_mmap(num, round_up): """Align the given integer number to the closest page offset, which usually is 4096 bytes. :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num / PAGESIZE) * PAGESIZE; + res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; if round_up and (res != num): - res += PAGESIZE; + res += ALLOCATIONGRANULARITY #END handle size return res; @@ -55,10 +55,10 @@ def ofs_end(self): def align(self): """Assures the previous window area is contained in the new one""" - nofs = align_to_page(self.ofs, 0) + nofs = align_to_mmap(self.ofs, 0) self.size += self.ofs - nofs # keep size constant self.ofs = nofs - self.size = align_to_page(self.size, 1) + self.size = align_to_mmap(self.size, 1) def extend_left_to(self, window, max_size): """Adjust the offset to start where the given window on our left ends if possible, From 4466476cf576cf5936a11524e67345a80e2ec5a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 10:48:28 +0200 Subject: [PATCH 0164/3719] Fixed missing ALLOCATIONGRANULARIY in python <2.6. Python is as unportable as ever across versions with simple functionality --- smmap/test/test_util.py | 1 - smmap/util.py | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 7caa427ba..4043cd83f 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,7 +1,6 @@ from lib import TestBase, FileCreator from smmap.util import * -from mmap import ALLOCATIONGRANULARITY import os import sys diff --git a/smmap/util.py b/smmap/util.py index e1f786538..667777861 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,7 +3,15 @@ import sys import mmap -from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ +from mmap import mmap, ACCESS_READ +try: + from mmap import ALLOCATIONGRANULARITY +except ImportError: + # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly + # useful for aligning the offset. The offset argument doesn't exist there though + from mmap import PAGESIZE as ALLOCATIONGRANULARITY +#END handle pythons missing quality assurance + from sys import getrefcount __all__ = [ "align_to_mmap", "is_64_bit", From 064aa81076b8b2e08209aa6525ca22065703b41d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 12:35:32 +0200 Subject: [PATCH 0165/3719] Implemented __len__ method in buffer, including small test. This has its caveats, but should be fine for responsible clients --- smmap/buf.py | 26 ++++++++++++++++++++++++-- smmap/test/test_buf.py | 5 ++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index b4f581316..4fc78ea6f 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -11,7 +11,10 @@ class MappedMemoryBuffer(object): The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access""" - __slots__ = '_c' # our cursor + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): @@ -20,6 +23,10 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! :param flags: Additional flags to be passed to os.open :raise ValueError: if the buffer could not achieve a valid state""" self._c = cursor @@ -30,6 +37,9 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): def __del__(self): self.end_access() + def __len__(self): + return self._size + def __getitem__(self, i): c = self._c assert c.is_valid() @@ -76,7 +86,18 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): - return self._c.use_region(offset, size, flags).is_valid() + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + #END handle size + self._size = size + #END set size + return res return False def end_access(self): @@ -85,6 +106,7 @@ def end_access(self): resources to be freed. Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 if self._c is not None: self._c.unuse_region() #END unuse region diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index c772e4999..48aeabb44 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -30,15 +30,18 @@ def test_basics(self): # can call end access any time buf.end_access() buf.end_access() + assert len(buf) == 0 # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True + assert len(buf) == fc.size assert buf.cursor().is_valid() # simple access @@ -63,7 +66,7 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 400 + max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), From d7b486df99139ff867db01f6427408a12ff213b3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:26:46 +0200 Subject: [PATCH 0166/3719] Added smmap as submodule, assured the sys path makes it available --- .gitmodules | 4 ++++ gitdb/__init__.py | 16 +++++++++------- gitdb/ext/smmap | 1 + 3 files changed, 14 insertions(+), 7 deletions(-) create mode 160000 gitdb/ext/smmap diff --git a/.gitmodules b/.gitmodules index 3db4c676d..1be8ccac9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = gitdb/ext/async url = git://github.com/gitpython-developers/async.git branch = master +[submodule "smmap"] + path = gitdb/ext/smmap + url = git://github.com/Byron/smmap.git + branch = master diff --git a/gitdb/__init__.py b/gitdb/__init__.py index a551f37dc..775c969cf 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -10,13 +10,15 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async')) - - try: - import async - except ImportError: - raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH") - #END verify import + for module in ('async', 'smmap'): + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + + try: + __import__(module) + except ImportError: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + #END verify import + #END handel imports #} END initialization diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap new file mode 160000 index 000000000..4466476cf --- /dev/null +++ b/gitdb/ext/smmap @@ -0,0 +1 @@ +Subproject commit 4466476cf576cf5936a11524e67345a80e2ec5a9 From d09158f9029c564c97cc33f173efd376eca873d1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:36:53 +0200 Subject: [PATCH 0167/3719] Made all types available in root package --- smmap/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/smmap/__init__.py b/smmap/__init__.py index 82cff638c..769858fa5 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -5,3 +5,7 @@ __homepage__ = "https://github.com/Byron/smmap" version_info = (0, 8, 0) __version__ = '.'.join(str(i) for i in version_info) + +# make everything available in root package for convenience +from mman import * +from buf import * From 9dc4a8dd154d15a243cd2d54f7d1631a913106f0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:42:05 +0200 Subject: [PATCH 0168/3719] Changed names to be more descriptive, hopefully. This opens op the option to implement such a manager differently, without the sliding window mechanics, which would be quite simple and not much better than a map of mmaps in the end --- smmap/buf.py | 6 +++--- smmap/mman.py | 40 ++++++++++++++++++++-------------------- smmap/test/test_buf.py | 14 +++++++------- smmap/test/test_mman.py | 12 ++++++------ smmap/test/test_util.py | 18 +++++++++--------- smmap/util.py | 16 ++++++++-------- 6 files changed, 53 insertions(+), 53 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 4fc78ea6f..94650a50c 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,11 +1,11 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import MemoryCursor +from mman import SlidingCursor import sys -__all__ = ["MappedMemoryBuffer"] +__all__ = ["SlidingWindowMapBuffer"] -class MappedMemoryBuffer(object): +class SlidingWindowMapBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. diff --git a/smmap/mman.py b/smmap/mman.py index 8e50a1445..312612298 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,8 +1,8 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" from util import ( - MemoryWindow, - MappedRegion, - MappedRegionList, + MapWindow, + MapRegion, + MapRegionList, is_64_bit, ) @@ -10,16 +10,16 @@ from weakref import ref import sys -__all__ = ["MappedMemoryManager"] +__all__ = ["SlidingWindowMapManager"] #{ Utilities #}END utilities -class MemoryCursor(object): +class SlidingCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed. - Cursors should not be created manually, but are instead returned by the MappedMemoryManager""" + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -29,8 +29,8 @@ class MemoryCursor(object): ) #{ Configuration - MemoryWindowCls = MemoryWindow - MappedRegionCls = MappedRegion + MapWindowCls = MapWindow + MapRegionCls = MapRegion #} END configuration def __init__(self, manager = None, regions = None): @@ -133,9 +133,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): #END while bisecting if existing_region is None: - left = self.MemoryWindowCls(0, 0) - mid = self.MemoryWindowCls(offset, size) - right = self.MemoryWindowCls(self.file_size(), 0) + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(self.file_size(), 0) # we want to honor the max memory size, and assure we have anough # memory available @@ -166,13 +166,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = self.MemoryWindowCls.from_region(a[insert_pos]) + right = self.MapWindowCls.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = self.MemoryWindowCls.from_region(a[insert_pos]) + right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window - left = self.MemoryWindowCls.from_region(a[insert_pos - 1]) + left = self.MapWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows mid.extend_left_to(left, window_size) @@ -189,7 +189,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -301,7 +301,7 @@ def fd(self): #} END interface -class MappedMemoryManager(object): +class SlidingWindowMapManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. Once a certain memory limit is reached globally, or if there cannot be more open file handles @@ -315,7 +315,7 @@ class MappedMemoryManager(object): space is full.""" __slots__ = [ - '_fdict', # mapping of path -> MappedRegionList + '_fdict', # mapping of path -> MapRegionList '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate '_max_handle_count', # maximum amount of handles to keep open @@ -324,7 +324,7 @@ class MappedMemoryManager(object): ] #{ Configuration - MappedRegionListCls = MappedRegionList + MapRegionListCls = MapRegionList #} END configuration _MB_in_bytes = 1024 * 1024 @@ -411,10 +411,10 @@ def make_cursor(self, path_or_fd): prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: - regions = self.MappedRegionListCls(path_or_fd) + regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return MemoryCursor(self, regions) + return SlidingCursor(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 48aeabb44..96ca4f882 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,6 +1,6 @@ from lib import TestBase, FileCreator -from smmap.mman import MappedMemoryManager +from smmap.mman import SlidingWindowMapManager from smmap.buf import * from random import randint @@ -9,8 +9,8 @@ import os -man_optimal = MappedMemoryManager() -man_worst_case = MappedMemoryManager( window_size=TestBase.k_window_test_size/100, +man_optimal = SlidingWindowMapManager() +man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, max_memory_size=TestBase.k_window_test_size/3, max_open_handles=15) @@ -21,10 +21,10 @@ def test_basics(self): # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size) # offset too large + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - buf = MappedMemoryBuffer() # can create uninitailized buffers + buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None # can call end access any time @@ -71,7 +71,7 @@ def test_basics(self): for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case')): - buf = MappedMemoryBuffer(manager.make_cursor(item)) + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e220f8bb2..dfe43e704 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MemoryCursor +from smmap.mman import SlidingCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -16,8 +16,8 @@ class TestMMan(TestBase): def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") - man = MappedMemoryManager() - ci = MemoryCursor(man) # invalid cursor + man = SlidingWindowMapManager() + ci = SlidingCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,10 +43,10 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - MemoryCursor(man)._destroy() + SlidingCursor(man)._destroy() def test_memory_manager(self): - man = MappedMemoryManager() + man = SlidingWindowMapManager() assert man.num_file_handles() == 0 assert man.num_open_files() == 0 assert man.window_size() > 0 @@ -82,7 +82,7 @@ def test_memman_operation(self): # small windows, a reasonable max memory. Not too many regions at once max_num_handles = 15 - man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) + man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) c = man.make_cursor(item) # still empty (more about that is tested in test_memory_manager() diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 4043cd83f..46de2ebaa 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -8,10 +8,10 @@ class TestMMan(TestBase): def test_window(self): - wl = MemoryWindow(0, 1) # left - wc = MemoryWindow(1, 1) # center - wc2 = MemoryWindow(10, 5) # another center - wr = MemoryWindow(8000, 50) # right + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right assert wl.ofs_end() == 1 assert wc.ofs_end() == 2 @@ -56,9 +56,9 @@ def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size / 2 rofs = align_to_mmap(4200, False) - rfull = MappedRegion(fc.path, 0, fc.size) - rhalfofs = MappedRegion(fc.path, rofs, fc.size) - rhalfsize = MappedRegion(fc.path, 0, half_size) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) # offsets assert rfull.ofs_begin() == 0 and rfull.size() == fc.size @@ -88,7 +88,7 @@ def test_region(self): assert rfull.usage_count() == 1 # window constructor - w = MemoryWindow.from_region(rfull) + w = MapWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): @@ -96,7 +96,7 @@ def test_region_list(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - ml = MappedRegionList(item) + ml = MapRegionList(item) assert ml.client_count() == 1 diff --git a/smmap/util.py b/smmap/util.py index 667777861..6ead86479 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -15,7 +15,7 @@ from sys import getrefcount __all__ = [ "align_to_mmap", "is_64_bit", - "MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"] + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -39,7 +39,7 @@ def is_64_bit(): #{ Utility Classes -class MemoryWindow(object): +class MapWindow(object): """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( 'ofs', # offset into the file in bytes @@ -51,7 +51,7 @@ def __init__(self, offset, size): self.size = size def __repr__(self): - return "MemoryWindow(%i, %i)" % (self.ofs, self.size) + return "MapWindow(%i, %i)" % (self.ofs, self.size) @classmethod def from_region(cls, region): @@ -84,7 +84,7 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class MappedRegion(object): +class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes :note: deallocates used region automatically on destruction""" __slots__ = [ @@ -145,7 +145,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): #END close file handle def __repr__(self): - return "MappedRegion<%i, %i>" % (self._b, self.size()) + return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface @@ -201,15 +201,15 @@ def includes_ofs(self, ofs): #} END interface -class MappedRegionList(list): - """List of MappedRegion instances associating a path with a list of regions.""" +class MapRegionList(list): + """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): - return super(MappedRegionList, cls).__new__(cls) + return super(MapRegionList, cls).__new__(cls) def __init__(self, path_or_fd): self._path_or_fd = path_or_fd From bc78951823623f9a529d0b515d85d6a0a1a1d8ac Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 15:31:07 +0200 Subject: [PATCH 0169/3719] moved code from cursor into manager, as it belongs there. This is a design error inherited from c++, but actually it makes overrides a bit harder, or lets say, less native, as the sliding mechanics where implemented in a class which is just the handle to a memory map in the end, which doesn't have to care about its allocation --- smmap/buf.py | 2 +- smmap/mman.py | 234 ++++++++++++++++++++-------------------- smmap/test/test_mman.py | 6 +- 3 files changed, 122 insertions(+), 120 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 94650a50c..741450303 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import SlidingCursor +from mman import MemoryCursor import sys diff --git a/smmap/mman.py b/smmap/mman.py index 312612298..16f878199 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -15,7 +15,7 @@ #}END utilities -class SlidingCursor(object): +class MemoryCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed. @@ -28,11 +28,6 @@ class SlidingCursor(object): '_size' # maximum size we should provide ) - #{ Configuration - MapWindowCls = MapWindow - MapRegionCls = MapRegion - #} END configuration - def __init__(self, manager = None, regions = None): self._manager = manager self._rlist = regions @@ -82,7 +77,7 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, flags = 0, _is_recursive=False): + def use_region(self, offset, size, flags = 0): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map @@ -104,115 +99,14 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # END handle existing region # END check existing region + # offset too large ? + if offset >= self._rlist.file_size(): + return self + #END handle offset + if need_region: - window_size = man._window_size - - # abort on offsets beyond our mapped file's size - currently we are invalid - if offset >= self.file_size(): - return self - # END handle offset too large - - # bisect to find an existing region. The c++ implementation cannot - # do that as it uses a linked list for regions. - existing_region = None - a = self._rlist - lo = 0 - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - ofs = a[mid]._b - if ofs <= offset: - if a[mid].includes_ofs(offset): - existing_region = a[mid] - break - #END have region - lo = mid+1 - else: - hi = mid - #END handle position - #END while bisecting - - if existing_region is None: - left = self.MapWindowCls(0, 0) - mid = self.MapWindowCls(offset, size) - right = self.MapWindowCls(self.file_size(), 0) - - # we want to honor the max memory size, and assure we have anough - # memory available - # Save calls ! - if self._manager._memory_size + window_size > self._manager._max_memory_size: - man._collect_lru_region(window_size) - #END handle collection - - # we assume the list remains sorted by offset - insert_pos = 0 - len_regions = len(a) - if len_regions == 1: - if a[0]._b <= offset: - insert_pos = 1 - #END maintain sort - else: - # find insert position - insert_pos = len_regions - for i, region in enumerate(a): - if region._b > offset: - insert_pos = i - break - #END if insert position is correct - #END for each region - # END obtain insert pos - - # adjust the actual offset and size values to create the largest - # possible mapping - if insert_pos == 0: - if len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side - else: - if insert_pos != len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - # END adjust right window - left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows - - mid.extend_left_to(left, window_size) - mid.extend_right_to(right, window_size) - mid.align() - - # it can happen that we align beyond the end of the file - if mid.ofs_end() > right.ofs: - mid.size = right.ofs - mid.ofs - #END readjust size - - # insert new region at the right offset to keep the order - try: - if man._handle_count >= man._max_handle_count: - raise Exception - #END assert own imposed max file handles - self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if _is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - man._collect_lru_region(0) - return self.use_region(offset, size, flags, True) - #END handle exceptions - - man._handle_count += 1 - man._memory_size += self._region.size() - a.insert(insert_pos, self._region) - else: - self._region = existing_region - #END need region handling - #END handle acquire region + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + #END need region handling self._region.increment_usage_count() self._ofs = offset - self._region._b @@ -301,6 +195,7 @@ def fd(self): #} END interface + class SlidingWindowMapManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. @@ -325,6 +220,8 @@ class SlidingWindowMapManager(object): #{ Configuration MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion #} END configuration _MB_in_bytes = 1024 * 1024 @@ -398,6 +295,111 @@ def _collect_lru_region(self, size): return num_found + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + #END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r + #{ Interface def make_cursor(self, path_or_fd): """:return: a cursor pointing to the given path or file descriptor. @@ -414,7 +416,7 @@ def make_cursor(self, path_or_fd): regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return SlidingCursor(self, regions) + return MemoryCursor(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index dfe43e704..79c6f9892 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import SlidingCursor +from smmap.mman import MemoryCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -17,7 +17,7 @@ def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") man = SlidingWindowMapManager() - ci = SlidingCursor(man) # invalid cursor + ci = MemoryCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,7 +43,7 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - SlidingCursor(man)._destroy() + MemoryCursor(man)._destroy() def test_memory_manager(self): man = SlidingWindowMapManager() From 03dd5ae25fe99b9b91212057ef57a09e40f23265 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 16:19:10 +0200 Subject: [PATCH 0170/3719] Changed design of memory managers to support different implementations. Currently there is a non-implemented static version, as well as the previous sliding window version. --- smmap/buf.py | 6 +- smmap/mman.py | 222 +++++++++++++++++++++++++++++--------------------- smmap/util.py | 4 +- 3 files changed, 134 insertions(+), 98 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 741450303..772a268e6 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,7 +10,11 @@ class SlidingWindowMapBuffer(object): memory of a mapped file. The mapping is controlled by the provided cursor. The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at the offset you used during initialization or begin_access""" + first byte at the offset you used during initialization or begin_access + + :note: Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" __slots__ = ( '_c', # our cursor '_size', # our supposed size diff --git a/smmap/mman.py b/smmap/mman.py index 16f878199..d799c77af 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -10,7 +10,7 @@ from weakref import ref import sys -__all__ = ["SlidingWindowMapManager"] +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] #{ Utilities #}END utilities @@ -125,11 +125,11 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset - to the window size. Please note that it might be smaller than you requested + to the window size. Please note that it might be smaller than you requested when calling use_region() :note: You can only obtain a buffer if this instance is_valid() ! :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) + return buffer(self._region.map(), self._ofs, self._size) def is_valid(self): """:return: True if we have a valid and usable region""" @@ -195,19 +195,17 @@ def fd(self): #} END interface +class StaticWindowMapManager(object): + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. -class SlidingWindowMapManager(object): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily - obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles - which result from each mmap call, the least recently used, and currently unused mapped regions - are unloaded automatically. + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. - :note: currently not thread-safe ! - :note: in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than - a safe amount of memory already, which would possibly cause memory allocations to fail as our address - space is full.""" + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + acomodate this fact""" __slots__ = [ '_fdict', # mapping of path -> MapRegionList @@ -222,11 +220,12 @@ class SlidingWindowMapManager(object): MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion + MemoryCursorCls = MemoryCursor #} END configuration _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. :param window_size: if 0, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size @@ -258,6 +257,8 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size + #{ Internal Methods + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full @@ -265,6 +266,117 @@ def _collect_lru_region(self, size): :raise RegionCollectionError: :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + raise NotImplementedError() + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + raise NotImplementedError() + + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """:return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + :note: if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + :note: Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.MemoryCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + :note: does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface + + + +class SlidingWindowMapManager(StaticWindowMapManager): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + :note: currently not thread-safe ! + :note: in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to 0""" + super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + + def _collect_lru_region(self, size): num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None @@ -296,10 +408,6 @@ def _collect_lru_region(self, size): return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, - see MapCursor.use_region. - :param a: A regions (a)rray - :return: The newly created region""" # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. r = None @@ -400,80 +508,4 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # END create new region return r - #{ Interface - def make_cursor(self, path_or_fd): - """:return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory - :note: if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - :note: Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" - regions = self._fdict.get(path_or_fd) - if regions is None: - regions = self.MapRegionListCls(path_or_fd) - self._fdict[path_or_fd] = regions - # END obtain region for path - return MemoryCursor(self, regions) - - def collect(self): - """Collect all available free-to-collect mapped regions - :return: Amount of freed handles""" - return self._collect_lru_region(0) - - def num_file_handles(self): - """:return: amount of file handles in use. Each mapped region uses one file handle""" - return self._handle_count - def num_open_files(self): - """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) - - def window_size(self): - """:return: size of each window when allocating new regions""" - return self._window_size - - def mapped_memory_size(self): - """:return: amount of bytes currently mapped in total""" - return self._memory_size - - def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" - return self._max_handle_count - - def max_mapped_memory_size(self): - """:return: maximum amount of memory we may allocate""" - return self._max_memory_size - - #} END interface - - #{ Special Purpose Interface - - def force_map_handle_removal_win(self, base_path): - """ONLY AVAILABLE ON WINDOWS - On windows removing files is not allowed if anybody still has it opened. - If this process is ourselves, and if the whole process uses this memory - manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to - allow the respective operation after all. - The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep - the cursors alive will not be using it anymore. They need to be recreated ! - :return: Amount of closed handles - :note: does nothing on non-windows platforms""" - if sys.platform != 'win32': - return - #END early bailout - - num_closed = 0 - for path, rlist in self._fdict.iteritems(): - if path.startswith(base_path): - for region in rlist: - region._mf.close() - num_closed += 1 - #END path matches - #END for each path - return num_closed - #} END special purpose interface diff --git a/smmap/util.py b/smmap/util.py index 6ead86479..21e285559 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -149,8 +149,8 @@ def __repr__(self): #{ Interface - def buffer(self): - """:return: a sliceable buffer which can be used to access the mapped memory""" + def map(self): + """:return: a memory map containing the memory""" return self._mf def ofs_begin(self): From 631b9ea2edfb005b1d48d55b35370f82ac2de196 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 17:47:48 +0200 Subject: [PATCH 0171/3719] Implemented static memory manager, for now without test --- smmap/buf.py | 2 +- smmap/mman.py | 98 ++++++++++++++++++++++++++++++++++++----- smmap/test/test_mman.py | 14 ++++-- 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 772a268e6..c4d252251 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import MemoryCursor +from mman import WindowCursor import sys diff --git a/smmap/mman.py b/smmap/mman.py index d799c77af..ba3a63f55 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -9,17 +9,23 @@ from exc import RegionCollectionError from weakref import ref import sys +from sys import getrefcount __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] #{ Utilities #}END utilities -class MemoryCursor(object): - """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed. + + +class WindowCursor(object): + """Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. - Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager""" + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + :note: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -130,7 +136,14 @@ def buffer(self): :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.map(), self._ofs, self._size) - + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None @@ -188,7 +201,7 @@ def fd(self): :note: it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): - return ValueError("File descriptor queried although mapping was generated from path") + raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() @@ -208,7 +221,7 @@ class StaticWindowMapManager(object): acomodate this fact""" __slots__ = [ - '_fdict', # mapping of path -> MapRegionList + '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate '_max_handle_count', # maximum amount of handles to keep open @@ -220,7 +233,7 @@ class StaticWindowMapManager(object): MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion - MemoryCursorCls = MemoryCursor + WindowCursorCls = WindowCursor #} END configuration _MB_in_bytes = 1024 * 1024 @@ -266,14 +279,77 @@ def _collect_lru_region(self, size): :raise RegionCollectionError: :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" - raise NotImplementedError() + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + for k, regions in self._fdict.iteritems(): + found_lonely_region = False + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): + # remove whole list + found_lonely_region = True + num_found += 1 + self._memory_size -= region.size() + self._handle_count -= 1 + self._fdict.pop(k) + + break + # END update lru_region + #END for each region + if found_lonely_region: + continue + # END skip iteration and restart + #END for each regions list + + # still here ? + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + #END while there is more memory to free + + return num_found + def _obtain_region(self, a, offset, size, flags, is_recursive): """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" - raise NotImplementedError() + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + # END handle array + + assert a.includes_ofs(offset) + assert a.includes_ofs(offset + size-1) + return r #}END internal methods @@ -293,7 +369,7 @@ def make_cursor(self, path_or_fd): regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return self.MemoryCursorCls(self, regions) + return self.WindowCursorCls(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 79c6f9892..e8266066c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MemoryCursor +from smmap.mman import WindowCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -17,7 +17,7 @@ def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") man = SlidingWindowMapManager() - ci = MemoryCursor(man) # invalid cursor + ci = WindowCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,7 +43,7 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - MemoryCursor(man)._destroy() + WindowCursor(man)._destroy() def test_memory_manager(self): man = SlidingWindowMapManager() @@ -65,11 +65,19 @@ def test_memory_manager(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): c = man.make_cursor(item) + assert c.path_or_fd() is item assert c.use_region(10, 10).is_valid() assert c.ofs_begin() == 10 assert c.size() == 10 assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error #END for each input + os.close(fd) def test_memman_operation(self): From e010084b0f40d7762f029a7ac4109c9bb99818a6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 18:28:37 +0200 Subject: [PATCH 0172/3719] test are running, once again, but not yet complete regarding the static manager --- smmap/mman.py | 92 ++++++++++++++--------------------------- smmap/test/test_mman.py | 71 +++++++++++++++++-------------- 2 files changed, 70 insertions(+), 93 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ba3a63f55..bdefe2e81 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -95,7 +95,8 @@ def use_region(self, offset, size, flags = 0): either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager - size = min(size, man.window_size()) # clamp size to window size + fsize = self._rlist.file_size() + size = min(size, man.window_size() or fsize) # clamp size to window size if self._region is not None: if self._region.includes_ofs(offset): @@ -106,7 +107,7 @@ def use_region(self, offset, size, flags = 0): # END check existing region # offset too large ? - if offset >= self._rlist.file_size(): + if offset >= fsize: return self #END handle offset @@ -238,10 +239,11 @@ class StaticWindowMapManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. - :param window_size: if 0, a default window size will be chosen depending on + :param window_size: if -1, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. @@ -254,7 +256,7 @@ def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handl self._memory_size = 0 self._handle_count = 0 - if window_size == 0: + if window_size < 0: coeff = 32 if is_64_bit(): coeff = 1024 @@ -281,43 +283,40 @@ def _collect_lru_region(self, size): :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): - for k, regions in self._fdict.iteritems(): - found_lonely_region = False + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and (lru_region is None or region._uc < lru_region._uc)): - # remove whole list - found_lonely_region = True - num_found += 1 - self._memory_size -= region.size() - self._handle_count -= 1 - self._fdict.pop(k) - - break + lru_region = region + lru_list = regions # END update lru_region #END for each region - if found_lonely_region: - continue - # END skip iteration and restart #END for each regions list - # still here ? - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary + if lru_region is None: + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 #END while there is more memory to free - return num_found - def _obtain_region(self, a, offset, size, flags, is_recursive): """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" - if self._memory_size + window_size > self._max_memory_size: - self._collect_lru_region(window_size) + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) #END handle collection r = None @@ -347,8 +346,8 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): self._memory_size += r.size() # END handle array - assert a.includes_ofs(offset) - assert a.includes_ofs(offset + size-1) + assert r.includes_ofs(offset) + assert r.includes_ofs(offset + size-1) return r #}END internal methods @@ -362,6 +361,8 @@ def make_cursor(self, path_or_fd): your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths are preferred unless you plan to keep the file descriptor open. + :note: file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. :note: Using file descriptors directly is faster once new windows are mapped as it prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) @@ -448,41 +449,10 @@ class SlidingWindowMapManager(StaticWindowMapManager): __slots__ = tuple() - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): - """Adjusts the default window size to 0""" + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - def _collect_lru_region(self, size): - num_found = 0 - while (size == 0) or (self._memory_size + size > self._max_memory_size): - lru_region = None - lru_list = None - for regions in self._fdict.itervalues(): - for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): - lru_region = region - lru_list = regions - # END update lru_region - #END for each region - #END for each regions list - - if lru_region is None: - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary - break - #END handle region not found - - num_found += 1 - del(lru_list[lru_list.index(lru_region)]) - self._memory_size -= lru_region.size() - self._handle_count -= 1 - #END while there is more memory to free - - return num_found - def _obtain_region(self, a, offset, size, flags, is_recursive): # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e8266066c..97866bbf9 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -46,40 +46,47 @@ def test_cursor(self): WindowCursor(man)._destroy() def test_memory_manager(self): - man = SlidingWindowMapManager() - assert man.num_file_handles() == 0 - assert man.num_open_files() == 0 - assert man.window_size() > 0 - assert man.mapped_memory_size() == 0 - assert man.max_mapped_memory_size() > 0 - - # collection doesn't raise in 'any' mode - man._collect_lru_region(0) - # doesn't raise if we are within the limit - man._collect_lru_region(10) - # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() - # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + #END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + # raises outside of limit + self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error + #END for each input + os.close(fd) + # END for each manager type - if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) - else: - self.failUnlessRaises(ValueError, c.fd) - #END handle value error - #END for each input - - os.close(fd) - def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") From 101a4d39108c4b4731760456523fd0f0d565b6f2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 19:50:43 +0200 Subject: [PATCH 0173/3719] Finally the mem manager and buffer tests run with the static one too --- smmap/mman.py | 18 ++-- smmap/test/test_buf.py | 6 +- smmap/test/test_mman.py | 216 ++++++++++++++++++++++------------------ 3 files changed, 131 insertions(+), 109 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index bdefe2e81..f1ce95d03 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,9 +4,9 @@ MapRegion, MapRegionList, is_64_bit, + align_to_mmap ) -from exc import RegionCollectionError from weakref import ref import sys from sys import getrefcount @@ -83,10 +83,10 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, flags = 0): + def use_region(self, offset, size = 0, flags = 0): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file - :param size: amount of bytes to map + :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. @@ -96,7 +96,7 @@ def use_region(self, offset, size, flags = 0): need_region = True man = self._manager fsize = self._rlist.file_size() - size = min(size, man.window_size() or fsize) # clamp size to window size + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size if self._region is not None: if self._region.includes_ofs(offset): @@ -246,6 +246,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, the manager will free as many handles as posisble""" @@ -278,8 +279,9 @@ def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region - :raise RegionCollectionError: :return: Amount of freed regions + :note: We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -297,9 +299,6 @@ def _collect_lru_region(self, size): #END for each regions list if lru_region is None: - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary break #END handle region not found @@ -344,10 +343,11 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): self._handle_count += 1 self._memory_size += r.size() + a.append(r) # END handle array assert r.includes_ofs(offset) - assert r.includes_ofs(offset + size-1) + #assert r.includes_ofs(offset+size-1) return r #}END internal methods diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 96ca4f882..d8b7fbcab 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,6 +1,6 @@ from lib import TestBase, FileCreator -from smmap.mman import SlidingWindowMapManager +from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager from smmap.buf import * from random import randint @@ -13,6 +13,7 @@ man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, max_memory_size=TestBase.k_window_test_size/3, max_open_handles=15) +static_man = StaticWindowMapManager() class TestBuf(TestBase): @@ -70,7 +71,8 @@ def test_basics(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case')): + (man_worst_case, 'worst case'), + (static_man, 'static optimial')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 97866bbf9..27be686ad 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -64,8 +64,9 @@ def test_memory_manager(self): man._collect_lru_region(0) # doesn't raise if we are within the limit man._collect_lru_region(10) - # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + # doesn't fail if we overallocate + assert man._collect_lru_region(sys.maxint) == 0 # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") @@ -92,103 +93,122 @@ def test_memman_operation(self): fc = FileCreator(self.k_window_test_size, "manager_operation_test") data = open(fc.path, 'rb').read() fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - max_num_handles = 15 - man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - assert c.size() == size - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) - - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # obtain second window, which spans the first part of the file - it is a still the same window - assert c.use_region(0, size-10).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == size-10 - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:size-10] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - size + overshoot - assert c.use_region(base_offset, size).is_valid() - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) + max_num_handles = 15 + #small_size = + for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) - # precondition - assert max_mapped_memory_size >= mapped_memory_size() - assert max_file_handles >= num_file_handles() + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() / 2 assert c.use_region(base_offset, size).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - #END for each item + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + + #assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + else: + assert rr().size() == fc.size + #END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + #END ignore static managers which only have one handle per file + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + #END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + #END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + # END for each manager type os.close(fd) From 82c97ea8f416bd99f501608b2aee8b7c4c5e78f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 20:12:40 +0200 Subject: [PATCH 0174/3719] Some more adjustments to make it work in all python versions --- smmap/mman.py | 2 +- smmap/util.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index f1ce95d03..fc6848413 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -136,7 +136,7 @@ def buffer(self): :note: You can only obtain a buffer if this instance is_valid() ! :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.map(), self._ofs, self._size) + return buffer(self._region.buffer(), self._ofs, self._size) def map(self): """ diff --git a/smmap/util.py b/smmap/util.py index 21e285559..80ba3c5b9 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -136,7 +136,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._size = len(self._mf) if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, size) + self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping finally: if isinstance(path_or_fd, basestring): @@ -148,7 +148,11 @@ def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface - + + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + def map(self): """:return: a memory map containing the memory""" return self._mf From a33e8d55d4d77d842edea94a78d801b23bb90294 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 21:59:22 +0200 Subject: [PATCH 0175/3719] Switched git db to the non-sliding version of the memory manager which is a good tradeoff between performance loss and resource handling --- gitdb/ext/smmap | 2 +- gitdb/pack.py | 76 +++++++++++++++++++++++-------------------------- gitdb/util.py | 13 +++++++++ 3 files changed, 50 insertions(+), 41 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 4466476cf..4cabaca18 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 4466476cf576cf5936a11524e67345a80e2ec5a9 +Subproject commit 4cabaca18cd30399288fc55863de412446ea51e7 diff --git a/gitdb/pack.py b/gitdb/pack.py index 7ae9786e6..0679a6ecf 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -10,10 +10,10 @@ ) from util import ( zlib, + mman, LazyMixin, unpack_from, bin_to_hex, - file_contents_ro_filepath, ) from fun import ( @@ -247,7 +247,7 @@ class PackIndexFile(LazyMixin): # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -261,22 +261,23 @@ def __init__(self, indexpath): def _set_cache_(self, attr): if attr == "_packfile_checksum": - self._packfile_checksum = self._data[-40:-20] + self._packfile_checksum = self._cursor.map()[-40:-20] elif attr == "_packfile_checksum": - self._packfile_checksum = self._data[-20:] - elif attr == "_data": + self._packfile_checksum = self._cursor.map()[-20:] + elif attr == "_cursor": # Note: We don't lock the file when reading as we cannot be sure # that we can actually write to the location - it could be a read-only # alternate for instance - self._data = file_contents_ro_filepath(self._indexpath) + self._cursor = mman.make_cursor(self._indexpath).use_region() else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties # CHECK VERSION - self._version = (self._data[:4] == self.index_v2_signature and 2) or 1 + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", self._data, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version @@ -297,16 +298,16 @@ def _set_cache_(self, attr): def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._data, 1024 + i*24) + (0, ) + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" - return unpack_from(">L", self._data, 1024 + i*24)[0] + return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] def _sha_v1(self, i): """see ``_sha_v2``""" base = 1024 + (i*24)+4 - return self._data[base:base+20] + return self._cursor.map()[base:base+20] def _crc_v1(self, i): """unsupported""" @@ -322,13 +323,13 @@ def _entry_v2(self, i): def _offset_v2(self, i): """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" - offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0] + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: - offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset return offset @@ -336,11 +337,11 @@ def _offset_v2(self, i): def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 - return self._data[base:base+20] + return self._cursor.map()[base:base+20] def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] #} END access V2 @@ -358,7 +359,7 @@ def _initialize(self): def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" - d = self._data + d = self._cursor.map() out = list() append = out.append for i in range(256): @@ -382,11 +383,11 @@ def path(self): def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" - return self._data[-40:-20] + return self._cursor.map()[-40:-20] def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" - return self._data[-20:] + return self._cursor.map()[-20:] def offsets(self): """:return: sequence of all offsets in the order in which they were written @@ -394,7 +395,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more if sys.byteorder == 'little': @@ -501,7 +502,7 @@ class PackFile(LazyMixin): for some reason - one clearly doesn't want to read 10GB at once in that case""" - __slots__ = ('_packpath', '_data', '_size', '_version') + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 @@ -513,26 +514,20 @@ def __init__(self, packpath): self._packpath = packpath def _set_cache_(self, attr): - if attr == '_data': - self._data = file_contents_ro_filepath(self._packpath) - - # read the header information - type_id, self._version, self._size = unpack_from(">LLL", self._data, 0) - - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - else: # must be '_size' or '_version' - # read header info - we do that just with a file stream - type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12)) - # END handle header + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - #END assert type id def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" - data = self._data + data = self._cursor.map() content_size = len(data) - self.footer_size cur_offset = start_offset or self.first_object_offset @@ -568,11 +563,11 @@ def data(self): """ :return: read-only data of this pack. It provides random access and usually is a memory map""" - return self._data + return self._cursor.map() def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._data[-20:] + return self._cursor.map()[-20:] def path(self): """:return: path to the packfile""" @@ -591,8 +586,9 @@ def collect_streams(self, offset): If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() + data = self._cursor.map() while True: - ostream = pack_object_at(self._data, offset, True)[1] + ostream = pack_object_at(data, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -614,14 +610,14 @@ def info(self, offset): :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, False)[1] + return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, True)[1] + return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """ @@ -704,7 +700,7 @@ def _object(self, sha, as_stream, index=-1): sha = self._index.sha(index) # END assure sha is present ( in output ) offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset)) if as_stream: if type_id not in delta_types: packstream = self._pack.stream(offset) diff --git a/gitdb/util.py b/gitdb/util.py index 4bb3c7352..4ce615585 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -23,6 +23,14 @@ # END try async zlib from async import ThreadPool +from smmap import ( + StaticWindowMapManager, + SlidingWindowMapBuffer + ) + +# initialize our global memory manager instance +# Use it to free cached (and unused) resources. +mman = StaticWindowMapManager() try: import hashlib @@ -180,6 +188,11 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): close(fd) # END assure file is closed +def sliding_ro_buffer(filepath, flags=0): + """:return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: From 34f01396b913220fe5b19e1f8e33f2d3f4ec2ce5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:05:36 +0200 Subject: [PATCH 0176/3719] Added changelog information --- doc/source/changes.rst | 5 +++++ gitdb/test/performance/test_pack.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 7b8ebecc6..999cc1309 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +***** +0.5.3 +***** +* Added support for smmap. SmartMMap allows resources to be managed and controlled. This brings the implementation closer to the way git handles memory maps, such that unused cached memory maps will automatically be freed once a resource limit is hit. The memory limit on 32 bit systems remains though as a sliding mmap implementation is not used for performance reasons. + ***** 0.5.2 ***** diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index da952b17a..20618024d 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -15,9 +15,11 @@ from time import time import random +from nose import SkipTest + class TestPackedDBPerformance(TestBigRepoR): - def _test_pack_random_access(self): + def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # sha lookup @@ -66,6 +68,7 @@ def _test_pack_random_access(self): print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) def test_correctness(self): + raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" From cb4059bccea6d01e41d63c66fda502823a35220f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:07:37 +0200 Subject: [PATCH 0177/3719] Bumped version info to 0.5.3 --- doc/source/conf.py | 4 ++-- setup.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 28deb3106..723a34503 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'GitDB' -copyright = u'2010, Sebastian Thiel' +copyright = u'2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.1' +release = '0.5.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3c6617422..86073971e 100755 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.2", + version = "0.5.3", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", @@ -80,7 +80,7 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.1)',), - install_requires='async >= 0.6.1', + requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), + install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""" ) From 09dd0eb9249493fc2d2897684035d753fc888acc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:22:42 +0200 Subject: [PATCH 0178/3719] A tiny win32 fix, once again --- smmap/test/test_util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 46de2ebaa..096c5f6df 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -71,9 +71,10 @@ def test_region(self): assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. - if sys.platform == 'win32': - assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0) - else: + # We only test on linux as it is inconsitent between the python versions + # as they use different mapping techniques to circumvent the missing offset + # argument of mmap. + if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) #END handle platforms From 84eedc5d1def7bfefefc729d09c39a6a9cde81f2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 14:54:17 +0200 Subject: [PATCH 0179/3719] Added option to help test cases to succeed on windows. Its for test systems only --- smmap/util.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index 80ba3c5b9..7ced28991 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -99,7 +99,13 @@ class MapRegion(object): if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot - + + #{ Configuration + # Used for testing only. If True, all data will be loaded into memory at once. + # This makes sure no file handles will remain open. + _test_read_into_memory = False + #} END configuration + def __init__(self, path_or_fd, ofs, size, flags = 0): """Initialize a region, allocate the memory map @@ -132,7 +138,13 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + if self._test_read_into_memory: + self._mf = self._read_into_memory(fd, ofs, actual_size) + else: + self._mf = mmap(fd, actual_size, **kwargs) + #END handle memory mode + self._size = len(self._mf) if self._need_compat_layer: @@ -144,6 +156,19 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): #END only close it if we opened it #END close file handle + def _read_into_memory(self, fd, offset, size): + """:return: string data as read from the given file descriptor, offset and size """ + os.lseek(fd, offset, os.SEEK_SET) + mf = '' + bytes_todo = size + while bytes_todo: + chunk = 1024*1024 + d = os.read(fd, chunk) + bytes_todo -= len(d) + mf += d + #END loop copy items + return mf + def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) @@ -204,7 +229,7 @@ def includes_ofs(self, ofs): #} END interface - + class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( From 65dacbbd74a46698932cccdcab54f7558d1da169 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 20:23:55 +0200 Subject: [PATCH 0180/3719] Fixed up configuration to create api documentation for the code. Improved some markup to be valid for sphinx. --- doc/source/conf.py | 2 +- smmap/mman.py | 5 +++-- smmap/util.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a0dac1166..90409a138 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../')) # -- General configuration ----------------------------------------------------- diff --git a/smmap/mman.py b/smmap/mman.py index c6efc9496..9629eca46 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -354,8 +354,9 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): #{ Interface def make_cursor(self, path_or_fd): - """:return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory :note: if a file descriptor is given, it is assumed to be open and valid, but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only diff --git a/smmap/util.py b/smmap/util.py index 7ced28991..07bdf7997 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -20,7 +20,9 @@ #{ Utilities def align_to_mmap(num, round_up): - """Align the given integer number to the closest page offset, which usually is 4096 bytes. + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" From 78cdb214fb6273169433fa662ad630afc26eb36a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 21:07:03 +0200 Subject: [PATCH 0181/3719] Wrote introduction, readme.rst now points to the introduction page to keep things consistent --- README.rst | 52 +--------------------------- doc/source/api.rst | 42 +++++++++++++++++++++++ doc/source/index.rst | 3 ++ doc/source/intro.rst | 82 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 51 deletions(-) mode change 100644 => 120000 README.rst create mode 100644 doc/source/api.rst create mode 100644 doc/source/intro.rst diff --git a/README.rst b/README.rst deleted file mode 100644 index 4c22eed2b..000000000 --- a/README.rst +++ /dev/null @@ -1,51 +0,0 @@ -#################### -Sliding MMap (smmap) -#################### -A straight forward implementation of a slidinging memory map. -The idea is that every access to a file goes through a memory map manager, which will on demand map a region of a file and provide a string-like object for reading. - -When reading from it, you will have to check whether you are still within your window boundary, and possibly obtain a new window as required. - -The great benefit of this system is that you can use it to map files of any size even on 32 bit systems. Additionally it will be able to close unused windows right away to return system resources. If there are multiple clients for the same file and location, the same window will be reused as well. - -As there is a global management facility, you are also able to forcibly free all open handles which is handy on windows, which would otherwise prevent the deletion of the involved files. - -For convenience, a stream class is provided which hides the usage of the memory manager behind a simple stream interface. - -************ -LIMITATIONS -************ -* The access is readonly by design. -* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0. - -************ -REQUIREMENTS -************ -* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function. - -******* -Install -******* -TODO - -****** -Source -****** -The source is available at git://github.com/Byron/smmap.git and can be cloned using:: - - git clone git://github.com/Byron/smmap.git - -************ -MAILING LIST -************ -http://groups.google.com/group/git-python - -************* -ISSUE TRACKER -************* -https://github.com/Byron/smmap/issues - -******* -LICENSE -******* -New BSD License diff --git a/README.rst b/README.rst new file mode 120000 index 000000000..7cafde78d --- /dev/null +++ b/README.rst @@ -0,0 +1 @@ +doc/source/intro.rst \ No newline at end of file diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..7e2854afa --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,42 @@ +.. _api-label: + +############# +API Reference +############# + +**************** +smmap.mman +**************** + +.. automodule:: smmap.mman + :members: + :undoc-members: + +**************** +smmap.buf +**************** + +.. automodule:: smmap.buf + :members: + :undoc-members: + +**************** +smmap.exc +**************** + +.. automodule:: smmap.exc + :members: + :undoc-members: + +**************** +smmap.util +**************** + +.. automodule:: smmap.util + :members: + :undoc-members: + + + + + diff --git a/doc/source/index.rst b/doc/source/index.rst index cb03044e0..d25ef8244 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,6 +12,9 @@ Contents: .. toctree:: :maxdepth: 2 + intro + tutorial + api changes Indices and tables diff --git a/doc/source/intro.rst b/doc/source/intro.rst new file mode 100644 index 000000000..b35e78569 --- /dev/null +++ b/doc/source/intro.rst @@ -0,0 +1,82 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +The interface also works around the missing offset parameter in python implementations up to python 2.5. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 2.4, 2.5 or 2.6 +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. +* It wasn't tested on python 2.7 and 3.x. + +############### +Getting Started +############### +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################ +Installing smmap +################ +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + + $ easy_install smmap + # or + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/Byron/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/Byron/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _pip: http://www.pip-installer.org/en/latest/ From a51b65d35d402791f774efe95d4b848cc524a403 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:48:55 +0200 Subject: [PATCH 0182/3719] Finished tutorial section, umproved capabilities of the buffer implementation to be more pythonic. Unfortunately, not all docs build yet because of some typical sphinx issue that results in an error which doesn't at all tell what the culprit actually is --- doc/source/intro.rst | 9 +-- doc/source/tutorial.rst | 118 ++++++++++++++++++++++++++++++++++++ smmap/buf.py | 10 +++ smmap/mman.py | 6 +- smmap/test/test_buf.py | 4 ++ smmap/test/test_tutorial.py | 83 +++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 doc/source/tutorial.rst create mode 100644 smmap/test/test_tutorial.py diff --git a/doc/source/intro.rst b/doc/source/intro.rst index b35e78569..30bff0ded 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -34,11 +34,6 @@ Limitations * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. * It wasn't tested on python 2.7 and 3.x. -############### -Getting Started -############### -It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. - ################ Installing smmap ################ @@ -53,7 +48,9 @@ As the command will install smmap in your respective python distribution, you wi If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: $ python setup.py install - + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + ################## Homepage and Links ################## diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 000000000..917b24594 --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,118 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying classes. + +****** +Design +****** +Per application, there is *MemoryManager* which is held as static instance and used throughout the application. It can be configured to keep your resources within certain limits. + +To access mapped regions, you require a cursor. Cursors point to exactly one file and serve as handles into it. As long as it exists, the respective memory region will remain available. + +For convenience, a buffer implementation is provided which handles cursors and resource allocation behind its simple buffer like interface. + +*************** +Memory Managers +*************** +There are two types of memory managers, one uses *static* windows, the other one uses *sliding* windows. A window is a region of a file mapped into memory. Although the names might be somewhat misleading as technically windows are always static, the *sliding* version will allocate relatively small windows whereas the *static* version will always map the whole file. + +The *static* manager does nothing more than keeping a client count on the respective memory maps which always map the whole file, which allows to make some assumptions that can lead to simplified data access and increased performance, but reduces the compatibility to 32 bit systems or giant files. + +The *sliding* memory manager therefore should be the default manager when preparing an application for handling huge amounts of data on 32 bit and 64 bit platforms:: + + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + mman.num_file_handles() + mman.mapped_memory_size() + # and many more ... + + +Cursors +******* +*Cursors* are handles that point onto a window, i.e. a region of a file mapped into memory. From them you may obtain a buffer through which the data of that window can actually be accessed:: + + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + +Now you would have to write your algorithms around this interface to properly slide through huge amounts of data. + +Alternatively you can use a convenience interface. + +******* +Buffers +******* +To make first use easier, at the expense of performance, there is a Buffer implementation which uses a cursor underneath. + +With it, you can access all data in a possibly huge file without having to take care of setting the cursor to different regions yourself:: + + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + +Disadvantages +************* +Buffers cannot be used in place of strings or maps, hence you have to slice them to have valid input for the sorts of struct and zlib. A slice means a lot of data handling overhead which makes buffers slower compared to using cursors directly. + diff --git a/smmap/buf.py b/smmap/buf.py index c4d252251..9b2402687 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -47,6 +47,8 @@ def __len__(self): def __getitem__(self, i): c = self._c assert c.is_valid() + if i < 0: + i = self._size + i if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage @@ -57,6 +59,12 @@ def __getslice__(self, i, j): # fast path, slice fully included - safes a concatenate operation and # should be the default assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxint: + j = self._size + if j < 0: + j = self._size + j if (c.ofs_begin() <= i) and (j < c.ofs_end()): b = c.ofs_begin() return c.buffer()[i-b:j-b] @@ -68,6 +76,7 @@ def __getslice__(self, i, j): md = str() while l: c.use_region(ofs, l) + assert c.is_valid() d = c.buffer()[:l] ofs += len(d) l -= len(d) @@ -102,6 +111,7 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): self._size = size #END set size return res + # END use our cursor return False def end_access(self): diff --git a/smmap/mman.py b/smmap/mman.py index 9629eca46..deba99809 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -11,17 +11,16 @@ import sys from sys import getrefcount -__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] #{ Utilities #}END utilities - class WindowCursor(object): """Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. - + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager :note: The current implementation is suited for static and sliding window managers, but it also means that it must be suited for the somewhat quite different sliding manager. It could be improved, but @@ -85,6 +84,7 @@ def assign(self, rhs): def use_region(self, offset = 0, size = 0, flags = 0): """Assure we point to a window which allows access to the given offset into the file + :param offset: absolute offset in bytes into the file :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d8b7fbcab..9881c6294 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -50,6 +50,10 @@ def test_basics(self): assert data[offset] == buf[0] assert data[offset:offset*2] == buf[0:offset] + # negative indices, partial slices + assert buf[-1] == buf[len(buf)-1] + assert buf[-10:] == buf[len(buf)-10:len(buf)] + # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py new file mode 100644 index 000000000..a9f4b1c08 --- /dev/null +++ b/smmap/test/test_tutorial.py @@ -0,0 +1,83 @@ +from lib import TestBase + +class TestTutorial(TestBase): + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + From ec511365bc641a320c66ce4e796918e58a1567c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:56:16 +0200 Subject: [PATCH 0183/3719] It turned out the :note: docstring was not supported. Now all documentation is being generated --- doc/source/api.rst | 24 ++++++++++++------------ smmap/mman.py | 39 ++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index 7e2854afa..cddd268c4 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -4,33 +4,33 @@ API Reference ############# -**************** -smmap.mman -**************** +*********************** +Mapped Memory Managers +*********************** .. automodule:: smmap.mman :members: :undoc-members: -**************** -smmap.buf -**************** +******* +Buffers +******* .. automodule:: smmap.buf :members: :undoc-members: -**************** -smmap.exc -**************** +********** +Exceptions +********** .. automodule:: smmap.exc :members: :undoc-members: -**************** -smmap.util -**************** +********* +Utilities +********* .. automodule:: smmap.util :members: diff --git a/smmap/mman.py b/smmap/mman.py index deba99809..15bb012b4 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -18,13 +18,15 @@ class WindowCursor(object): - """Pointer into the mapped region of the memory manager, keeping the map + """ + Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - :note: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but - I see no real need to do so.""" + + **Note**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -91,8 +93,9 @@ def use_region(self, offset = 0, size = 0, flags = 0): for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file - :note: The size actually mapped may be smaller than the given size. If that is the case, - either the file has reached its end, or the map was created between two existing regions""" + + **note**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager fsize = self._rlist.file_size() @@ -123,9 +126,10 @@ def use_region(self, offset = 0, size = 0, flags = 0): def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - :note: the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it - helps to free up resource more quickly""" + + **note** the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" self._region = None # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! @@ -133,9 +137,11 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - :note: You can only obtain a buffer if this instance is_valid() ! - :note: buffers should not be cached passed the duration of your access as it will - prevent resources from being freed even though they might not be accounted for anymore !""" + + **note** You can only obtain a buffer if this instance is_valid() ! + + **note** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) def map(self): @@ -155,7 +161,8 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - :note: only if is_valid() is True""" + + **note** only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): @@ -177,7 +184,8 @@ def region_ref(self): def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - :note: cursor must be valid for this to work""" + + **note** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) @@ -199,7 +207,8 @@ def path(self): def fd(self): """:return: file descriptor used to create the underlying mapping. - :note: it is not required to be valid anymore + + **note** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): raise ValueError("File descriptor queried although mapping was generated from path") From 59cd3da62b5038783deeb2883262682758ec1eec Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:58:44 +0200 Subject: [PATCH 0184/3719] Made README.rst a copy of intro.rst. unfortunately symlinks are not followed by github. This is a real issue to me ... --- README.rst | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) mode change 120000 => 100644 README.rst diff --git a/README.rst b/README.rst deleted file mode 120000 index 7cafde78d..000000000 --- a/README.rst +++ /dev/null @@ -1 +0,0 @@ -doc/source/intro.rst \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..30bff0ded --- /dev/null +++ b/README.rst @@ -0,0 +1,79 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +The interface also works around the missing offset parameter in python implementations up to python 2.5. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 2.4, 2.5 or 2.6 +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. +* It wasn't tested on python 2.7 and 3.x. + +################ +Installing smmap +################ +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + + $ easy_install smmap + # or + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/Byron/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/Byron/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _pip: http://www.pip-installer.org/en/latest/ From f097bd611a82289d6bb95074fdf596332cb1c980 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:10:08 +0200 Subject: [PATCH 0185/3719] Fixed wrong operating system fields --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e2bb622fd..2e97e59c6 100755 --- a/setup.py +++ b/setup.py @@ -41,8 +41,8 @@ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", - "Operating System :: Windows", - "Operating System :: OSX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", ], long_description=long_description, From cf297b7b81bc5f6011c49d818d776ed7915fa1ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:33:49 +0200 Subject: [PATCH 0186/3719] Removed possibly invalid documentation tags --- smmap/buf.py | 6 +++--- smmap/mman.py | 53 +++++++++++++++++++++++++++++---------------------- smmap/util.py | 3 ++- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 9b2402687..00ddbacd9 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -12,9 +12,9 @@ class SlidingWindowMapBuffer(object): The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access - :note: Although this type effectively hides the fact that there are mapped windows - underneath, it can unfortunately not be used in any non-pure python method which - needs a buffer or string""" + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" __slots__ = ( '_c', # our cursor '_size', # our supposed size diff --git a/smmap/mman.py b/smmap/mman.py index 15bb012b4..9b08ae969 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -24,7 +24,7 @@ class WindowCursor(object): Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - **Note**: The current implementation is suited for static and sliding window managers, but it also means + **Note:**: The current implementation is suited for static and sliding window managers, but it also means that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( @@ -94,7 +94,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file - **note**: The size actually mapped may be smaller than the given size. If that is the case, + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager @@ -127,7 +127,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - **note** the cursor unuses the region automatically upon destruction. It is recommended + **Note:** the cursor unuses the region automatically upon destruction. It is recommended to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None @@ -138,9 +138,9 @@ def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - **note** You can only obtain a buffer if this instance is_valid() ! + **Note:** You can only obtain a buffer if this instance is_valid() ! - **note** buffers should not be cached passed the duration of your access as it will + **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) @@ -162,7 +162,7 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - **note** only if is_valid() is True""" + **Note:** only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): @@ -185,7 +185,7 @@ def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - **note** cursor must be valid for this to work""" + **Note:** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) @@ -208,7 +208,7 @@ def path(self): def fd(self): """:return: file descriptor used to create the underlying mapping. - **note** it is not required to be valid anymore + **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): raise ValueError("File descriptor queried although mapping was generated from path") @@ -289,9 +289,11 @@ def _collect_lru_region(self, size): :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :return: Amount of freed regions - :note: We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. - :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None @@ -366,15 +368,18 @@ def make_cursor(self, path_or_fd): """ :return: a cursor pointing to the given path or file descriptor. It can be used to map new regions of the file into memory - :note: if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - :note: file descriptors are problematic as they are not necessarily unique, as two - different files opened and closed in succession might have the same file descriptor id. - :note: Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: regions = self.MapRegionListCls(path_or_fd) @@ -426,7 +431,8 @@ def force_map_handle_removal_win(self, base_path): This really may only be used if you know that the items which keep the cursors alive will not be using it anymore. They need to be recreated ! :return: Amount of closed handles - :note: does nothing on non-windows platforms""" + + **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return #END early bailout @@ -451,8 +457,9 @@ class SlidingWindowMapManager(StaticWindowMapManager): which result from each mmap call, the least recently used, and currently unused mapped regions are unloaded automatically. - :note: currently not thread-safe ! - :note: in the current implementation, we will automatically unload windows if we either cannot + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot create more memory maps (as the open file handles limit is hit) or if we have allocated more than a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" diff --git a/smmap/util.py b/smmap/util.py index 07bdf7997..b0fd83b3f 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -88,7 +88,8 @@ def extend_right_to(self, window, max_size): class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes - :note: deallocates used region automatically on destruction""" + + **Note:** deallocates used region automatically on destruction""" __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) From 4524faf0d0c5383268b134084954b34faeaa766d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:29:22 +0200 Subject: [PATCH 0187/3719] Fixed up docs for upcoming release. Bumped version to 0.5.3 --- Makefile | 2 +- gitdb/__init__.py | 7 +++++++ gitdb/db/base.py | 7 ++++--- gitdb/db/mem.py | 4 ++-- gitdb/db/pack.py | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 15 +++++++++------ gitdb/pack.py | 29 +++++++++++++++++------------ gitdb/stream.py | 6 +++--- gitdb/test/db/lib.py | 2 +- gitdb/util.py | 28 ++++++++++++++++++---------- setup.py | 9 +++++---- 12 files changed, 69 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index e65c55a6d..c6c159bdc 100644 --- a/Makefile +++ b/Makefile @@ -23,5 +23,5 @@ clean:: rm -f *.so coverage:: build - PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=dulwich --with-coverage --cover-erase --cover-inclusive gitdb + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=gitdb --with-coverage --cover-erase --cover-inclusive gitdb diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 775c969cf..91359105c 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -24,6 +24,13 @@ def _init_externals(): _init_externals() +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/gitdb" +version_info = (0, 5, 3) +__version__ = '.'.join(str(i) for i in version_info) + + # default imports from db import * from base import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2189d4193..984acafbf 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -72,7 +72,8 @@ def stream_async(self, reader): :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` :return: async.Reader yielding OStream|InvalidOStream instances in any order - :note: depending on the system configuration, it might not be possible to + + **Note:** depending on the system configuration, it might not be possible to read all OStreams at once. Instead, read them individually using reader.read(x) where x is small enough.""" # base implementation just uses the stream method repeatedly @@ -140,7 +141,7 @@ def store_async(self, reader): The same instances will be used in the output channel as were received in by the Reader. - :note:As some ODB implementations implement this operation atomic, they might + **Note:** As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" # base implementation uses store to perform the work @@ -158,7 +159,7 @@ def __init__(self, root_path): """Initialize this instance to look for its files at the given root path All subsequent operations will be relative to this path :raise InvalidDBRoot: - :note: The base will not perform any accessablity checking as the base + **Note:** The base will not perform any accessablity checking as the base might not yet be accessible, but become accessible before the first access.""" super(FileDBBase, self).__init__() diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 8012ad15e..5d76c83cc 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -33,7 +33,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): it to the actual physical storage, as it allows to query whether object already exists in the target storage before introducing actual IO - :note: memory is currently not threadsafe, hence the async methods cannot be used + **Note:** memory is currently not threadsafe, hence the async methods cannot be used for storing""" def __init__(self): @@ -92,7 +92,7 @@ def sha_iter(self): def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly - :note: the object will only be written if it did not exist in the target db + **Note:** the object will only be written if it did not exist in the target db :return: amount of streams actually copied into odb. If smaller than the amount of input shas, one or more objects did already exist in odb""" count = 0 diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eef3f712e..4c9d0b919 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -58,7 +58,7 @@ def _pack_info(self, sha): """:return: tuple(entity, index) for an item at the given sha :param sha: 20 or 40 byte sha :raise BadObject: - :note: This method is not thread-safe, but may be hit in multi-threaded + **Note:** This method is not thread-safe, but may be hit in multi-threaded operation. The worst thing that can happen though is a counter that was not incremented, or the list being in wrong order. So we safe the time for locking here, lets see how that goes""" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 84eedc5d1..f097bd611 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 84eedc5d1def7bfefefc729d09c39a6a9cde81f2 +Subproject commit f097bd611a82289d6bb95074fdf596332cb1c980 diff --git a/gitdb/fun.py b/gitdb/fun.py index 5bbe8efc3..66130ebee 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -138,7 +138,7 @@ def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than absofs. - :note: global method for performance only, it belongs to DeltaChunkList""" + **Note:** global method for performance only, it belongs to DeltaChunkList""" lo = 0 hi = len(dcl) while lo < hi: @@ -414,9 +414,11 @@ def pack_object_header_info(data): return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): - """:return: string defining the pack header comprised of the object type - and its incompressed size in bytes - :parmam obj_type: pack type_id of the object + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte hdr = str() # output string @@ -483,7 +485,7 @@ def stream_copy(read, write, size, chunk_size): Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size - :note: its much like stream_copy utility, but operates just using methods""" + **Note:** its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written # WRITE ALL DATA UP TO SIZE @@ -597,7 +599,8 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes - :note: transcribed to python from the similar routine in patch-delta.c""" + + **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf while i < delta_buf_size: diff --git a/gitdb/pack.py b/gitdb/pack.py index 0679a6ecf..d840441e9 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -173,7 +173,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): class IndexWriter(object): """Utility to cache index information, allowing to write all information later in one go to the given stream - :note: currently only writes v2 indices""" + **Note:** currently only writes v2 indices""" __slots__ = '_objs' def __init__(self): @@ -391,7 +391,8 @@ def indexfile_checksum(self): def offsets(self): """:return: sequence of all offsets in the order in which they were written - :note: return value can be random accessed, but may be immmutable""" + + **Note:** return value can be random accessed, but may be immmutable""" if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears @@ -497,10 +498,10 @@ class PackFile(LazyMixin): packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - :note: at some point, this might be implemented using streams as well, or - streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that - case""" + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' @@ -625,8 +626,9 @@ def stream_iter(self, start_offset=0): to access the data in the pack directly. :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. - :note: Iterating a pack directly is costly as the datastream has to be decompressed - to determine the bounds between the objects""" + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) #} END Read-Database like Interface @@ -902,9 +904,11 @@ def write_pack(cls, object_iter, pack_write, index_write=None, :param zlib_compression: the zlib compression level to use :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack and over all contents of the index. If index_write was None, index_binsha will be None - :note: The destination of the write functions is up to the user. It could - be a socket, or a file for instance - :note: writes only undeltified objects""" + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" objs = object_iter if not object_count: if not isinstance(object_iter, (tuple, list)): @@ -979,7 +983,8 @@ def create(cls, object_iter, base_dir, object_count = None, zlib_compression = z and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files :return: PackEntity instance initialized with the new pack - :note: for more information on the other parameters see the write_pack method""" + + **Note:** for more information on the other parameters see the write_pack method""" pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) diff --git a/gitdb/stream.py b/gitdb/stream.py index 8010a0551..632213c27 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -51,7 +51,7 @@ class DecompressMemMapReader(LazyMixin): To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. - :note: The chunk-size should be carefully selected as it will involve quite a bit + **Note:** The chunk-size should be carefully selected as it will involve quite a bit of string copying due to the way the zlib is implemented. Its very wasteful, hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here @@ -609,8 +609,8 @@ class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor - :note: operates on raw file descriptors - :note: for this to work, you have to use the close-method of this instance""" + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") # default exception diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 416c8c588..4af4483c7 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -65,7 +65,7 @@ def _assert_object_writing_simple(self, db): def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW - :note: requires write access to the database""" + **Note:** requires write access to the database""" # start in 'dry-run' mode, using a simple sha1 writer ostreams = (ZippedStoreShaWriter, None) for ostreamcls in ostreams: diff --git a/gitdb/util.py b/gitdb/util.py index 4ce615585..23784de3f 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -119,8 +119,9 @@ def __getslice__(self, start, end): #{ Routines def make_sha(source=''): - """A python2.4 workaround for the sha/hashlib module fiasco - :note: From the dulwich project """ + """A python2.4 workaround for the sha/hashlib module fiasco + + **Note** From the dulwich project """ try: return hashlib.sha1(source) except NameError: @@ -146,6 +147,7 @@ def allocate_memory(size): def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd + :param fd: file descriptor opened for reading :param stream: if False, random access is provided, otherwise the stream interface is provided. @@ -173,14 +175,16 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible + :return: random access compatible memory of the given filepath :param stream: see ``file_contents_ro`` :param allow_mmap: see ``file_contents_ro`` :param flags: additional flags to pass to os.open :raise OSError: If the file could not be opened - :note: for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object - databases anyway, and they use it with the help of the ``flags`` parameter""" + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: return file_contents_ro(fd, stream, allow_mmap) @@ -189,7 +193,8 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): # END assure file is closed def sliding_ro_buffer(filepath, flags=0): - """:return: a buffer compatible object which uses our mapped memory manager internally + """ + :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) @@ -254,7 +259,7 @@ class LockedFD(object): This type handles error correctly in that it will assure a consistent state on destruction. - :note: with this setup, parallel reading is not possible""" + **note** with this setup, parallel reading is not possible""" __slots__ = ("_filepath", '_fd', '_write') def __init__(self, filepath): @@ -283,7 +288,8 @@ def open(self, write=False, stream=False): and must not be closed directly :raise IOError: if the lock could not be retrieved :raise OSError: If the actual file could not be opened for reading - :note: must only be called once""" + + **note** must only be called once""" if self._write is not None: raise AssertionError("Called %s multiple times" % self.open) @@ -327,13 +333,15 @@ def commit(self): """When done writing, call this function to commit your changes into the actual file. The file descriptor will be closed, and the lockfile handled. - :note: can be called multiple times""" + + **Note** can be called multiple times""" self._end_writing(successful=True) def rollback(self): """Abort your operation without any changes. The file descriptor will be closed, and the lock released. - :note: can be called multiple times""" + + **Note** can be called multiple times""" self._end_writing(successful=False) def _end_writing(self, successful=True): diff --git a/setup.py b/setup.py index 86073971e..b5c8c7046 100755 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from distutils.command.build_ext import build_ext import os, sys +import gitdb as meta # wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None @@ -69,11 +70,11 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.3", + version = meta.__version__, description = "Git Object Database", - author = "Sebastian Thiel", - author_email = "byronimo@gmail.com", - url = "http://gitorious.org/git-python/gitdb", + author = meta.__author__, + author_email = meta.__contact__, + url = meta.__homepage__, packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':'gitdb'}, From a5ed410aa0d3bed587214c3c017af2916b740da1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 3 Jul 2011 13:39:19 +0200 Subject: [PATCH 0188/3719] removed test suite from being distributed. It didn't work properly anyway and I am not going to dig into the setup tools mess --- setup.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index b5c8c7046..7924b5a07 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from distutils.command.build_ext import build_ext import os, sys -import gitdb as meta # wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None @@ -68,15 +67,23 @@ def get_data_files(self): setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too +# NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot +# satisfy the dependencies at installation time, unfortunately, due to inherent limitations +# of distutils, which cannot install the prerequesites of a package before the acutal package. +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/gitdb" +version_info = (0, 5, 3) +__version__ = '.'.join(str(i) for i in version_info) + setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = meta.__version__, + version = __version__, description = "Git Object Database", - author = meta.__author__, - author_email = meta.__contact__, - url = meta.__homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), - package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + author = __author__, + author_email = __contact__, + url = __homepage__, + packages = ('gitdb', 'gitdb.db'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", From aea587d9b414d7f150922c2923a1b9394d0d0543 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 08:58:22 +0200 Subject: [PATCH 0189/3719] Added license info for packs --- LICENSE | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/LICENSE b/LICENSE index be11e73c1..0d6fe8bdb 100644 --- a/LICENSE +++ b/LICENSE @@ -28,3 +28,15 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Additional Licenses +------------------- +The files at +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx +and +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack +are licensed under GNU GPL as part of the git source repository, +see http://en.wikipedia.org/wiki/Git_%28software%29 for more information. + +They are not required for the actual operation, which is why they are not found +in the distribution package. From 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:36:30 +0200 Subject: [PATCH 0190/3719] Fixed possible bug as a method was called using an old signature. Apparently this code branch never ran in the tests --- smmap/mman.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9b08ae969..ef9d43df0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -68,7 +68,7 @@ def _copy_from(self, rhs): self._size = rhs._size if self._region is not None: - self._region.increment_usage_count(1) + self._region.increment_usage_count() # END handle regions def __copy__(self): @@ -358,7 +358,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # END handle array assert r.includes_ofs(offset) - #assert r.includes_ofs(offset+size-1) return r #}END internal methods From 0e64168dd3f43b02857e60183d40c86480f01dc7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 15:03:16 +0200 Subject: [PATCH 0191/3719] pack: updated to use its cursor properly, which will be required if huge packs should be handled. This reduces performance as each access requires the windows to be checked/adjusted, but that is how it is. This should be circumvented using other backends, like the one of the gitcmd or libgit2. Default is now the sliding memory map manager --- gitdb/pack.py | 32 +++++++++++++++++++------------- gitdb/util.py | 3 ++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index d840441e9..c6d1cc313 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -73,7 +73,7 @@ #{ Utilities -def pack_object_at(data, offset, as_stream): +def pack_object_at(cursor, offset, as_stream): """ :return: Tuple(abs_data_offset, PackInfo|PackStream) an object of the correct type according to the type_id of the object. @@ -83,7 +83,7 @@ def pack_object_at(data, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - data = buffer(data, offset) + data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None @@ -269,6 +269,10 @@ def _set_cache_(self, attr): # that we can actually write to the location - it could be a read-only # alternate for instance self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) + #END assert window size else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -528,13 +532,13 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" - data = self._cursor.map() - content_size = len(data) - self.footer_size + c = self._cursor + content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset null = NullStream() while cur_offset < content_size: - data_offset, ostream = pack_object_at(data, cur_offset, True) + data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset @@ -563,12 +567,14 @@ def version(self): def data(self): """ :return: read-only data of this pack. It provides random access and usually - is a memory map""" - return self._cursor.map() + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.map()[-20:] + return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] def path(self): """:return: path to the packfile""" @@ -587,9 +593,9 @@ def collect_streams(self, offset): If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() - data = self._cursor.map() + c = self._cursor while True: - ostream = pack_object_at(data, offset, True)[1] + ostream = pack_object_at(c, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -611,14 +617,14 @@ def info(self, offset): :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1] + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1] + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """ @@ -702,7 +708,7 @@ def _object(self, sha, as_stream, index=-1): sha = self._index.sha(index) # END assure sha is present ( in output ) offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) if as_stream: if type_id not in delta_types: packstream = self._pack.stream(offset) diff --git a/gitdb/util.py b/gitdb/util.py index 23784de3f..e96c133e8 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -25,12 +25,13 @@ from async import ThreadPool from smmap import ( StaticWindowMapManager, + SlidingWindowMapManager, SlidingWindowMapBuffer ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = StaticWindowMapManager() +mman = SlidingWindowMapManager() try: import hashlib From ef5dc3d968b3aeed16a02ec705f89b72ad46fa84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:40:54 +0200 Subject: [PATCH 0192/3719] Optimized test_pack_streaming not to cache the objects anymore. Instead an iterator is provided which does the job. Previously it would easily use 750 MB of ram to keep all the associated objects, more than 350k. Still a lot of memory for just 350k objects, but its python after all --- gitdb/ext/smmap | 2 +- gitdb/test/performance/test_pack_streaming.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f097bd611..9c3eb3daf 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f097bd611a82289d6bb95074fdf596332cb1c980 +Subproject commit 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 795ed1e26..3c40ed0fb 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -40,10 +40,9 @@ def test_pack_writing(self): count = 0 total_size = 0 st = time() - objs = list() for sha in pdb.sha_iter(): count += 1 - objs.append(pdb.stream(sha)) + pdb.stream(sha) if count == ni: break #END gather objects for pack-writing @@ -51,7 +50,7 @@ def test_pack_writing(self): print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) st = time() - PackEntity.write_pack(objs, ostream.write) + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) From a4deb8461e6fbca0306a24f22b0c494679ad4757 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:45:39 +0200 Subject: [PATCH 0193/3719] wrote change log for next release. Choosing memory manager type based on the actual python version for best efficiency --- doc/source/changes.rst | 5 +++++ gitdb/util.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 999cc1309..839bf16a8 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +***** +0.5.4 +***** +* Adjusted implementation to use the SlidingMemoryManager by default in python 2.6 for efficiency reasons. In Python 2.4, the StaticMemoryManager will be used instead. + ***** 0.5.3 ***** diff --git a/gitdb/util.py b/gitdb/util.py index e96c133e8..013f5fc78 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -31,7 +31,11 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = SlidingWindowMapManager() +if sys.version_info[1] < 6: + mman = StaticWindowMapManager() +else: + mman = SlidingWindowMapManager() +#END handle mman try: import hashlib From d13e3aeb168645965720ddec1f469e05771563ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:55:32 +0200 Subject: [PATCH 0194/3719] updated changelog, bumped version --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ee17e0af0..03148fb31 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +********** +v0.8.1 +********** +- A single bugfix + + ********** v0.8.0 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 769858fa5..ae6e72eba 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 0) +version_info = (0, 8, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 656a2e0b4da7d60ac638d1615751a89efb3a4eee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 17:00:27 +0200 Subject: [PATCH 0195/3719] bumped version to 0.5.4 --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 91359105c..800b292da 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 3) +version_info = (0, 5, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 9c3eb3daf..d13e3aeb1 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b +Subproject commit d13e3aeb168645965720ddec1f469e05771563ef diff --git a/setup.py b/setup.py index 7924b5a07..62bc6d007 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 3) +version_info = (0, 5, 4) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, From e2b170a80462255dcc2003380db3547f55d8f14d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 8 Jul 2011 08:13:00 -0400 Subject: [PATCH 0196/3719] Workaround for #1 --- smmap/mman.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ef9d43df0..f5b7efbd1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -52,11 +52,14 @@ def _destroy(self): if self._rlist is not None: # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: - # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path_or_fd()) - #END remove regions list from manager + try: + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except TypeError: + pass #END handle regions def _copy_from(self, rhs): From bdc1258abb48328a389720df2ffc404692add426 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 29 Aug 2011 21:45:59 +0200 Subject: [PATCH 0197/3719] Added LICENSE file containing a copy of (new)BSD --- LICENSE | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..710010f1f --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the async project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + From 40fd4f31ab594dcfe049032c62ec61d2f0c3e492 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Jan 2012 23:09:53 +0100 Subject: [PATCH 0198/3719] Added some more in-code comments to clarify why that exception is caught --- smmap/mman.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/smmap/mman.py b/smmap/mman.py index f5b7efbd1..7b0984358 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -59,7 +59,12 @@ def _destroy(self): self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager except TypeError: + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount alltogether. pass + #END exception handling #END handle regions def _copy_from(self, rhs): From 360a8956fe73a0a96315e946f52737569d990369 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Jan 2012 23:12:02 +0100 Subject: [PATCH 0199/3719] Bumped version to 0.8.2 --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index ae6e72eba..a10cd5c99 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 1) +version_info = (0, 8, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From e96d2c381ef06667726eb745c67357de9d2a88fb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Jul 2012 20:56:03 +0200 Subject: [PATCH 0200/3719] Submodules now use the http protocol to facilitate checkout in corporate networks --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5dfd8e993..062ec5ba1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = git://github.com/gitpython-developers/async.git + url = http://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap - url = git://github.com/Byron/smmap.git + url = http://github.com/Byron/smmap.git From ec6998f503e4619cd6bdecbbf372552ea126900a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Jul 2012 21:12:03 +0200 Subject: [PATCH 0201/3719] Updated submodules to latest version --- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/ext/async b/gitdb/ext/async index 10310824c..039c1d5c2 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 10310824c001deab8fea85b88ebda0696f964b3e +Subproject commit 039c1d5c26bc2ceaa9e55082efae2068d9873e45 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index d13e3aeb1..360a8956f 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit d13e3aeb168645965720ddec1f469e05771563ef +Subproject commit 360a8956fe73a0a96315e946f52737569d990369 From 0328caa516fffdbb5f28fd59798a9775aa2b05f5 Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 2 Nov 2012 10:43:48 +1100 Subject: [PATCH 0202/3719] Update gitmodules to point to the https location of the git repositories. Signed-off-by: David --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 062ec5ba1..978105388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = http://github.com/gitpython-developers/async.git + url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap - url = http://github.com/Byron/smmap.git + url = https://github.com/Byron/smmap.git From 2ea761425b9fa1fda57e83a877f4e2fdc336a9a3 Mon Sep 17 00:00:00 2001 From: Jason Schadel Date: Mon, 19 Nov 2012 16:27:46 -0500 Subject: [PATCH 0203/3719] Remove requires in setup.py. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 1c58cb656..a1dad3642 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,6 @@ def _stamp_version(filename): package_data = {'git.test' : ['fixtures/*']}, package_dir = {'git':'git'}, license = "BSD License", - requires=('gitdb (>=0.5.1)',), install_requires='gitdb >= 0.5.1', zip_safe=False, long_description = """\ From 5991698ee2b3046bbc9cfc3bd2abd3a881f514dd Mon Sep 17 00:00:00 2001 From: "Marcus R. Brown" Date: Fri, 11 Jan 2013 13:43:49 -0700 Subject: [PATCH 0204/3719] Support repos that use the .git-file mechanism. --- git/repo/base.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/git/repo/base.py b/git/repo/base.py index 20c96b228..df52137eb 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -71,6 +71,7 @@ class Repo(object): re_hexsha_shortened = re.compile('^[0-9A-Fa-f]{4,40}$') re_author_committer_start = re.compile(r'^(author|committer)') re_tab_full_line = re.compile(r'^\t(.*)$') + re_git_file_gitdir = re.compile('gitdir: (.*)') # invariants # represents the configuration level of a configuration file @@ -113,6 +114,17 @@ def __init__(self, path=None, odbt = DefaultDBType): self.git_dir = gitpath self._working_tree_dir = curpath break + if isfile(gitpath): + line = open(gitpath, 'r').readline().strip() + match = self.re_git_file_gitdir.match(line) + if match: + gitpath = match.group(1) + if not os.path.isabs(gitpath): + gitpath = os.path.normpath(join(curpath, gitpath)) + if is_git_dir(gitpath): + self.git_dir = gitpath + self._working_tree_dir = curpath + break curpath, dummy = os.path.split(curpath) if not dummy: break From 3621c06c3173bff395645bd416f0efafa20a1da6 Mon Sep 17 00:00:00 2001 From: "Marcus R. Brown" Date: Fri, 11 Jan 2013 13:47:06 -0700 Subject: [PATCH 0205/3719] Add tests for .git-file. --- git/test/fixtures/git_file | 1 + git/test/test_repo.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 git/test/fixtures/git_file diff --git a/git/test/fixtures/git_file b/git/test/fixtures/git_file new file mode 100644 index 000000000..2efda9f50 --- /dev/null +++ b/git/test/fixtures/git_file @@ -0,0 +1 @@ +gitdir: ./.real diff --git a/git/test/test_repo.py b/git/test/test_repo.py index 18d5c1b84..a4d148d18 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -594,6 +594,23 @@ def test_repo_odbtype(self): target_type = GitCmdObjectDB assert isinstance(self.rorepo.odb, target_type) + @with_rw_repo('HEAD') + def test_git_file(self, rwrepo): + # Move the .git directory to another location and create the .git file. + real_path_abs = os.path.abspath(join_path_native(rwrepo.working_tree_dir, '.real')) + os.rename(rwrepo.git_dir, real_path_abs) + git_file_path = join_path_native(rwrepo.working_tree_dir, '.git') + open(git_file_path, 'wb').write(fixture('git_file')) + + # Create a repo and make sure it's pointing to the relocated .git directory. + git_file_repo = Repo(rwrepo.working_tree_dir) + assert os.path.abspath(git_file_repo.git_dir) == real_path_abs + + # Test using an absolute gitdir path in the .git file. + open(git_file_path, 'wb').write('gitdir: %s\n' % real_path_abs) + git_file_repo = Repo(rwrepo.working_tree_dir) + assert os.path.abspath(git_file_repo.git_dir) == real_path_abs + def test_submodules(self): assert len(self.rorepo.submodules) == 1 # non-recursive assert len(list(self.rorepo.iter_submodules())) >= 2 From 007bd4b8190a6e85831c145e0aed5c68594db556 Mon Sep 17 00:00:00 2001 From: Igor Bondarenko Date: Thu, 14 Feb 2013 15:01:39 +0200 Subject: [PATCH 0206/3719] Fixed parse_actor_and_date with mangled tags --- git/objects/util.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/git/objects/util.py b/git/objects/util.py index 4c9323b85..af46f3c61 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -167,6 +167,7 @@ def parse_date(string_date): # precompiled regex _re_actor_epoch = re.compile(r'^.+? (.*) (\d+) ([+-]\d+).*$') +_re_only_actor = re.compile(r'^.+? (.*)$') def parse_actor_and_date(line): """Parse out the actor (author or committer) info from a line like:: @@ -174,8 +175,13 @@ def parse_actor_and_date(line): author Tom Preston-Werner 1191999972 -0700 :return: [Actor, int_seconds_since_epoch, int_timezone_offset]""" + actor, epoch, offset = '', 0, 0 m = _re_actor_epoch.search(line) - actor, epoch, offset = m.groups() + if m: + actor, epoch, offset = m.groups() + else: + m = _re_only_actor.search(line) + actor = m.group(1) if m else line or '' return (Actor._from_string(actor), int(epoch), utctz_to_altz(offset)) From 53b65e074e4d62ea5d0251b37c35fd055e403110 Mon Sep 17 00:00:00 2001 From: niyaton Date: Mon, 25 Feb 2013 01:22:30 +0900 Subject: [PATCH 0207/3719] Added support for separeted git dir. --- git/repo/base.py | 6 ++++++ git/repo/fun.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/git/repo/base.py b/git/repo/base.py index 20c96b228..7dcf409dc 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -32,6 +32,7 @@ from fun import ( rev_parse, is_git_dir, + read_gitfile, touch ) @@ -113,6 +114,11 @@ def __init__(self, path=None, odbt = DefaultDBType): self.git_dir = gitpath self._working_tree_dir = curpath break + gitpath = read_gitfile(gitpath) + if gitpath: + self.git_dir = gitpath + self._working_tree_dir = curpath + break curpath, dummy = os.path.split(curpath) if not dummy: break diff --git a/git/repo/fun.py b/git/repo/fun.py index 03d557164..86d3c6a99 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -30,6 +30,17 @@ def is_git_dir(d): os.readlink(headref).startswith('refs')) return False +def read_gitfile(f): + """ This is taken from the git setup.c:read_gitfile function. + :return gitdir path or None if gitfile is invalid.""" + + if not isfile(f): + return None + line = open(f, 'r').readline().rstrip() + if line[0:8] != 'gitdir: ': + return None + path = os.path.realpath(line[8:]) + return path if is_git_dir(path) else None def short_to_long(odb, hexsha): """:return: long hexadecimal sha1 from the given less-than-40 byte hexsha From db82455bd91ce00c22f6ee2b0dc622f117f07137 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Thu, 11 Apr 2013 18:39:03 +0000 Subject: [PATCH 0208/3719] [#6078] #102 Work-around mergetag blocks by ignoring them --- git/objects/commit.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index fd4187b08..8e74f8bfa 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -426,11 +426,18 @@ def _deserialize(self, stream): self.committer, self.committed_date, self.committer_tz_offset = parse_actor_and_date(readline()) + # we might run into one or more mergetag blocks, skip those for now + next_line = readline() + while next_line.startswith('mergetag '): + next_line = readline() + while next_line.startswith(' '): + next_line = readline() + # now we can have the encoding line, or an empty line followed by the optional # message. self.encoding = self.default_encoding # read encoding or empty line to separate message - enc = readline() + enc = next_line enc = enc.strip() if enc: self.encoding = enc[enc.find(' ')+1:] From f122a6aa3eb386914faa58ef3bf336f27b02fab0 Mon Sep 17 00:00:00 2001 From: Tim Van Steenburgh Date: Wed, 17 Apr 2013 18:21:53 +0000 Subject: [PATCH 0209/3719] Return bytes if object name can't be utf8-decoded Signed-off-by: Tim Van Steenburgh --- git/objects/fun.py | 10 +++++++--- git/test/test_fun.py | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/git/objects/fun.py b/git/objects/fun.py index 9b0a377cb..8c3806444 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -70,9 +70,13 @@ def tree_entries_from_data(data): # default encoding for strings in git is utf8 # Only use the respective unicode object if the byte stream was encoded name = data[ns:i] - name_enc = name.decode("utf-8") - if len(name) > len(name_enc): - name = name_enc + try: + name_enc = name.decode("utf-8") + except UnicodeDecodeError: + pass + else: + if len(name) > len(name_enc): + name = name_enc # END handle encoding # byte is NULL, get next 20 diff --git a/git/test/test_fun.py b/git/test/test_fun.py index b7991cdbe..36435ae4d 100644 --- a/git/test/test_fun.py +++ b/git/test/test_fun.py @@ -249,3 +249,8 @@ def test_tree_traversal_single(self): entries = traverse_tree_recursive(odb, commit.tree.binsha, '') assert entries # END for each commit + +def test_tree_entries_from_data(): + from git.objects.fun import tree_entries_from_data + r = tree_entries_from_data(b'100644 \x9f\0aaa') + assert r == [('aaa', 33188, '\x9f')], r From 5869c5c1a51d448a411ae0d51d888793c35db9c0 Mon Sep 17 00:00:00 2001 From: Tim Van Steenburgh Date: Wed, 17 Apr 2013 18:43:19 +0000 Subject: [PATCH 0210/3719] Fix whacky indentation Signed-off-by: Tim Van Steenburgh --- git/objects/fun.py | 14 +++++++------- git/test/test_fun.py | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/git/objects/fun.py b/git/objects/fun.py index 8c3806444..e2d4df558 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -70,13 +70,13 @@ def tree_entries_from_data(data): # default encoding for strings in git is utf8 # Only use the respective unicode object if the byte stream was encoded name = data[ns:i] - try: - name_enc = name.decode("utf-8") - except UnicodeDecodeError: - pass - else: - if len(name) > len(name_enc): - name = name_enc + try: + name_enc = name.decode("utf-8") + except UnicodeDecodeError: + pass + else: + if len(name) > len(name_enc): + name = name_enc # END handle encoding # byte is NULL, get next 20 diff --git a/git/test/test_fun.py b/git/test/test_fun.py index 36435ae4d..bbd5d1597 100644 --- a/git/test/test_fun.py +++ b/git/test/test_fun.py @@ -251,6 +251,6 @@ def test_tree_traversal_single(self): # END for each commit def test_tree_entries_from_data(): - from git.objects.fun import tree_entries_from_data - r = tree_entries_from_data(b'100644 \x9f\0aaa') - assert r == [('aaa', 33188, '\x9f')], r + from git.objects.fun import tree_entries_from_data + r = tree_entries_from_data(b'100644 \x9f\0aaa') + assert r == [('aaa', 33188, '\x9f')], r From d3a728277877924e889e9fef42501127f48a4e77 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Wed, 9 Oct 2013 19:02:56 +0000 Subject: [PATCH 0211/3719] [#5330] Ensure wait() is called on git processes --- git/cmd.py | 1 + git/objects/commit.py | 3 +++ git/remote.py | 17 ++++------------- git/repo/base.py | 7 +++++-- git/util.py | 12 ++++++++++++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 63a7134e0..75687a416 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -80,6 +80,7 @@ def __del__(self): # try to kill it try: os.kill(self.proc.pid, 2) # interrupt signal + self.proc.wait() # ensure process goes away except AttributeError: # try windows # for some reason, providing None for stdout/stderr still prints something. This is why diff --git a/git/objects/commit.py b/git/objects/commit.py index 8e74f8bfa..0565b2c0b 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -8,6 +8,7 @@ Actor, Iterable, Stats, + finalize_process ) from git.diff import Diffable from tree import Tree @@ -251,6 +252,8 @@ def _iter_from_process_or_stream(cls, repo, proc_or_stream): assert len(hexsha) == 40, "Invalid line: %s" % hexsha yield Commit(repo, hex_to_bin(hexsha)) # END for each line in stream + if has_attr(proc_or_stream, 'wait'): + finalize_process(proc_or_stream) @classmethod diff --git a/git/remote.py b/git/remote.py index 5e4439fb1..e38b3540d 100644 --- a/git/remote.py +++ b/git/remote.py @@ -24,7 +24,10 @@ TagReference ) -from git.util import join_path +from git.util import ( + join_path, + finalize_process + ) from gitdb.util import join import re @@ -58,18 +61,6 @@ def digest_process_messages(fh, progress): # END while file is not done reading return dropped_lines -def finalize_process(proc): - """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" - try: - proc.wait() - except GitCommandError,e: - # if a push has rejected items, the command has non-zero return status - # a return status of 128 indicates a connection error - reraise the previous one - if proc.poll() == 128: - raise - pass - # END exception handling - def add_progress(kwargs, git, progress): """Add the --progress flag to the given kwargs dict if supported by the git command. If the actual progress in the given progress instance is not diff --git a/git/repo/base.py b/git/repo/base.py index 14efabdc6..0bc3c12cf 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -6,7 +6,10 @@ from git.exc import InvalidGitRepositoryError, NoSuchPathError from git.cmd import Git -from git.util import Actor +from git.util import ( + Actor, + finalize_process + ) from git.refs import * from git.index import IndexFile from git.objects import * @@ -14,7 +17,6 @@ from git.remote import ( Remote, digest_process_messages, - finalize_process, add_progress ) @@ -541,6 +543,7 @@ def untracked_files(self): untracked_files.append(untracked_info.replace("#\t", "").rstrip()) # END for each utracked info line # END for each line + finalize_process(proc) return untracked_files @property diff --git a/git/util.py b/git/util.py index a9e87d6f6..130d77628 100644 --- a/git/util.py +++ b/git/util.py @@ -121,6 +121,18 @@ def get_user_id(): # END get username from login return "%s@%s" % (username, platform.node()) +def finalize_process(proc): + """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" + try: + proc.wait() + except GitCommandError,e: + # if a push has rejected items, the command has non-zero return status + # a return status of 128 indicates a connection error - reraise the previous one + if proc.poll() == 128: + raise + pass + # END exception handling + #} END utilities #{ Classes From c6b08c27a031f8b8b0bb6c41747ca1bc62b72706 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Thu, 17 Oct 2013 15:33:59 +0000 Subject: [PATCH 0212/3719] [#5330] Fixed has_attr typo --- git/objects/commit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 0565b2c0b..4ccd9d755 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -252,7 +252,7 @@ def _iter_from_process_or_stream(cls, repo, proc_or_stream): assert len(hexsha) == 40, "Invalid line: %s" % hexsha yield Commit(repo, hex_to_bin(hexsha)) # END for each line in stream - if has_attr(proc_or_stream, 'wait'): + if hasattr(proc_or_stream, 'wait'): finalize_process(proc_or_stream) From 3f277ba01f9a93fb040a365eef80f46ce6a9de85 Mon Sep 17 00:00:00 2001 From: BoppreH Date: Thu, 17 Oct 2013 21:47:55 -0300 Subject: [PATCH 0213/3719] Avoid spawning console windows when running from .pyw By adding `shell=True,` to the list of Popen parameters, we avoid spawning console windows when scripts call this method from a windowless (.pyw) Python script. --- git/cmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/cmd.py b/git/cmd.py index 576a5300a..579fbc83a 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -336,6 +336,7 @@ def execute(self, command, stderr=PIPE, stdout=PIPE, close_fds=(os.name=='posix'),# unsupported on linux + shell=True, **subprocess_kwargs ) if as_process: From 2e6957abf8cd88824282a19b74497872fe676a46 Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Tue, 28 Jan 2014 21:07:31 -0500 Subject: [PATCH 0214/3719] Fix missed import from d3a7282 The `finalize_process` method was moved but this import wasn't carried with it. --- git/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/util.py b/git/util.py index 130d77628..0a533e508 100644 --- a/git/util.py +++ b/git/util.py @@ -13,6 +13,8 @@ import tempfile import platform +from exc import GitCommandError + from gitdb.util import ( make_sha, LockedFD, From 03097c7ace28c5516aacbb1617265e50a9043a84 Mon Sep 17 00:00:00 2001 From: Maxim Syabro Date: Mon, 10 Feb 2014 01:58:27 +0800 Subject: [PATCH 0215/3719] Fixed NameError --- git/refs/reference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/reference.py b/git/refs/reference.py index 29d051a6f..284f4c9ac 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -18,7 +18,7 @@ def require_remote_ref_path(func): """A decorator raising a TypeError if we are not a valid remote, based on the path""" def wrapper(self, *args): if not self.path.startswith(self._remote_common_path_default + "/"): - raise ValueError("ref path does not point to a remote reference: %s" % path) + raise ValueError("ref path does not point to a remote reference: %s" % self.path) return func(self, *args) #END wrapper wrapper.__name__ = func.__name__ From 1b3ab5598e93369282502d049d64cb2ca12839cb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Feb 2014 20:50:30 +0100 Subject: [PATCH 0216/3719] tabs to spaces --- setup.py | 0 smmap/buf.py | 246 ++++---- smmap/exc.py | 6 +- smmap/mman.py | 1122 +++++++++++++++++------------------ smmap/test/lib.py | 96 +-- smmap/test/test_buf.py | 204 +++---- smmap/test/test_mman.py | 400 ++++++------- smmap/test/test_tutorial.py | 160 ++--- smmap/test/test_util.py | 212 +++---- smmap/util.py | 464 +++++++-------- 10 files changed, 1455 insertions(+), 1455 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/smmap/buf.py b/smmap/buf.py index 00ddbacd9..255c6b54d 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -6,129 +6,129 @@ __all__ = ["SlidingWindowMapBuffer"] class SlidingWindowMapBuffer(object): - """A buffer like object which allows direct byte-wise object and slicing into - memory of a mapped file. The mapping is controlled by the provided cursor. - - The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at the offset you used during initialization or begin_access - - **Note:** Although this type effectively hides the fact that there are mapped windows - underneath, it can unfortunately not be used in any non-pure python method which - needs a buffer or string""" - __slots__ = ( - '_c', # our cursor - '_size', # our supposed size - ) - - - def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): - """Initalize the instance to operate on the given cursor. - :param cursor: if not None, the associated cursor to the file you want to access - If None, you have call begin_access before using the buffer and provide a cursor - :param offset: absolute offset in bytes - :param size: the total size of the mapping. Defaults to the maximum possible size - From that point on, the __len__ of the buffer will be the given size or the file size. - If the size is larger than the mappable area, you can only access the actually available - area, although the length of the buffer is reported to be your given size. - Hence it is in your own interest to provide a proper size ! - :param flags: Additional flags to be passed to os.open - :raise ValueError: if the buffer could not achieve a valid state""" - self._c = cursor - if cursor and not self.begin_access(cursor, offset, size, flags): - raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") - # END handle offset + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by the provided cursor. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at the offset you used during initialization or begin_access + + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) + + + def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset - def __del__(self): - self.end_access() - - def __len__(self): - return self._size - - def __getitem__(self, i): - c = self._c - assert c.is_valid() - if i < 0: - i = self._size + i - if not c.includes_ofs(i): - c.use_region(i, 1) - # END handle region usage - return c.buffer()[i-c.ofs_begin()] - - def __getslice__(self, i, j): - c = self._c - # fast path, slice fully included - safes a concatenate operation and - # should be the default - assert c.is_valid() - if i < 0: - i = self._size + i - if j == sys.maxint: - j = self._size - if j < 0: - j = self._size + j - if (c.ofs_begin() <= i) and (j < c.ofs_end()): - b = c.ofs_begin() - return c.buffer()[i-b:j-b] - else: - l = j-i # total length - ofs = i - # Keeping tokens in a list could possible be faster, but the list - # overhead outweighs the benefits (tested) ! - md = str() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - md += d - #END while there are bytes to read - return md - # END fast or slow path - #{ Interface - - def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): - """Call this before the first use of this instance. The method was already - called by the constructor in case sufficient information was provided. - - For more information no the parameters, see the __init__ method - :param path: if cursor is None the existing one will be used. - :return: True if the buffer can be used""" - if cursor: - self._c = cursor - #END update our cursor - - # reuse existing cursors if possible - if self._c is not None and self._c.is_associated(): - res = self._c.use_region(offset, size, flags).is_valid() - if res: - # if given size is too large or default, we computer a proper size - # If its smaller, we assume the combination between offset and size - # as chosen by the user is correct and use it ! - # If not, the user is in trouble. - if size > self._c.file_size(): - size = self._c.file_size() - offset - #END handle size - self._size = size - #END set size - return res - # END use our cursor - return False - - def end_access(self): - """Call this method once you are done using the instance. It is automatically - called on destruction, and should be called just in time to allow system - resources to be freed. - - Once you called end_access, you must call begin access before reusing this instance!""" - self._size = 0 - if self._c is not None: - self._c.unuse_region() - #END unuse region - - def cursor(self): - """:return: the currently set cursor which provides access to the data""" - return self._c - - #}END interface + def __del__(self): + self.end_access() + + def __len__(self): + return self._size + + def __getitem__(self, i): + c = self._c + assert c.is_valid() + if i < 0: + i = self._size + i + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + return c.buffer()[i-c.ofs_begin()] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxint: + j = self._size + if j < 0: + j = self._size + j + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i-b:j-b] + else: + l = j-i # total length + ofs = i + # Keeping tokens in a list could possible be faster, but the list + # overhead outweighs the benefits (tested) ! + md = str() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + md += d + #END while there are bytes to read + return md + # END fast or slow path + #{ Interface + + def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if cursor is None the existing one will be used. + :return: True if the buffer can be used""" + if cursor: + self._c = cursor + #END update our cursor + + # reuse existing cursors if possible + if self._c is not None and self._c.is_associated(): + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + #END handle size + self._size = size + #END set size + return res + # END use our cursor + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 + if self._c is not None: + self._c.unuse_region() + #END unuse region + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/exc.py b/smmap/exc.py index a090d24d5..f0ed7dcd8 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -1,7 +1,7 @@ """Module with system exceptions""" class MemoryManagerError(Exception): - """Base class for all exceptions thrown by the memory manager""" - + """Base class for all exceptions thrown by the memory manager""" + class RegionCollectionError(MemoryManagerError): - """Thrown if a memory region could not be collected, or if no region for collection was found""" + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index 7b0984358..97c42c5bb 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,11 +1,11 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" from util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - align_to_mmap - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + align_to_mmap + ) from weakref import ref import sys @@ -18,564 +18,564 @@ class WindowCursor(object): - """ - Pointer into the mapped region of the memory manager, keeping the map - alive until it is destroyed and no other client uses it. + """ + Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. - Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - - **Note:**: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but - I see no real need to do so.""" - __slots__ = ( - '_manager', # the manger keeping all file regions - '_rlist', # a regions list with regions for our file - '_region', # our current region or None - '_ofs', # relative offset from the actually mapped area to our start area - '_size' # maximum size we should provide - ) - - def __init__(self, manager = None, regions = None): - self._manager = manager - self._rlist = regions - self._region = None - self._ofs = 0 - self._size = 0 - - def __del__(self): - self._destroy() - - def _destroy(self): - """Destruction code to decrement counters""" - self.unuse_region() - - if self._rlist is not None: - # Actual client count, which doesn't include the reference kept by the manager, nor ours - # as we are about to be deleted - try: - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: - # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path_or_fd()) - # END remove regions list from manager - except TypeError: - # sometimes, during shutdown, getrefcount is None. Its possible - # to re-import it, however, its probably better to just ignore - # this python problem (for now). - # The next step is to get rid of the error prone getrefcount alltogether. - pass - #END exception handling - #END handle regions - - def _copy_from(self, rhs): - """Copy all data from rhs into this instance, handles usage count""" - self._manager = rhs._manager - self._rlist = rhs._rlist - self._region = rhs._region - self._ofs = rhs._ofs - self._size = rhs._size - - if self._region is not None: - self._region.increment_usage_count() - # END handle regions - - def __copy__(self): - """copy module interface""" - cpy = type(self)() - cpy._copy_from(self) - return cpy - - #{ Interface - def assign(self, rhs): - """Assign rhs to this instance. This is required in order to get a real copy. - Alternativly, you can copy an existing instance using the copy module""" - self._destroy() - self._copy_from(rhs) - - def use_region(self, offset = 0, size = 0, flags = 0): - """Assure we point to a window which allows access to the given offset into the file - - :param offset: absolute offset in bytes into the file - :param size: amount of bytes to map. If 0, all available bytes will be mapped - :param flags: additional flags to be given to os.open in case a file handle is initially opened - for mapping. Has no effect if a region can actually be reused. - :return: this instance - it should be queried for whether it points to a valid memory region. - This is not the case if the mapping failed becaues we reached the end of the file - - **Note:**: The size actually mapped may be smaller than the given size. If that is the case, - either the file has reached its end, or the map was created between two existing regions""" - need_region = True - man = self._manager - fsize = self._rlist.file_size() - size = min(size or fsize, man.window_size() or fsize) # clamp size to window size - - if self._region is not None: - if self._region.includes_ofs(offset): - need_region = False - else: - self.unuse_region() - # END handle existing region - # END check existing region - - # offset too large ? - if offset >= fsize: - return self - #END handle offset - - if need_region: - self._region = man._obtain_region(self._rlist, offset, size, flags, False) - #END need region handling - - self._region.increment_usage_count() - self._ofs = offset - self._region._b - self._size = min(size, self._region.ofs_end() - offset) - - return self - - def unuse_region(self): - """Unuse the ucrrent region. Does nothing if we have no current region - - **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it - helps to free up resource more quickly""" - self._region = None - # note: should reset ofs and size, but we spare that for performance. Its not - # allowed to query information if we are not valid ! + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" + __slots__ = ( + '_manager', # the manger keeping all file regions + '_rlist', # a regions list with regions for our file + '_region', # our current region or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) + + def __init__(self, manager = None, regions = None): + self._manager = manager + self._rlist = regions + self._region = None + self._ofs = 0 + self._size = 0 + + def __del__(self): + self._destroy() + + def _destroy(self): + """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + try: + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except TypeError: + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount alltogether. + pass + #END exception handling + #END handle regions + + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = rhs._rlist + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + if self._region is not None: + self._region.increment_usage_count() + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy + + #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternativly, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset = 0, size = 0, flags = 0): + """Assure we point to a window which allows access to the given offset into the file + + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map. If 0, all available bytes will be mapped + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed becaues we reached the end of the file + + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + fsize = self._rlist.file_size() + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size + + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + # offset too large ? + if offset >= fsize: + return self + #END handle offset + + if need_region: + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + #END need region handling + + self._region.increment_usage_count() + self._ofs = offset - self._region._b + self._size = min(size, self._region.ofs_end() - offset) + + return self + + def unuse_region(self): + """Unuse the ucrrent region. Does nothing if we have no current region + + **Note:** the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! - def buffer(self): - """Return a buffer object which allows access to our memory region from our offset - to the window size. Please note that it might be smaller than you requested when calling use_region() - - **Note:** You can only obtain a buffer if this instance is_valid() ! - - **Note:** buffers should not be cached passed the duration of your access as it will - prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) - - def map(self): - """ - :return: the underlying raw memory map. Please not that the offset and size is likely to be different - to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole - file in case of StaticWindowMapManager""" - return self._region.map() - - def is_valid(self): - """:return: True if we have a valid and usable region""" - return self._region is not None - - def is_associated(self): - """:return: True if we are associated with a specific file already""" - return self._rlist is not None - - def ofs_begin(self): - """:return: offset to the first byte pointed to by our cursor - - **Note:** only if is_valid() is True""" - return self._region._b + self._ofs - - def ofs_end(self): - """:return: offset to one past the last available byte""" - # unroll method calls for performance ! - return self._region._b + self._ofs + self._size - - def size(self): - """:return: amount of bytes we point to""" - return self._size - - def region_ref(self): - """:return: weak ref to our mapped region. - :raise AssertionError: if we have no current region. This is only useful for debugging""" - if self._region is None: - raise AssertionError("region not set") - return ref(self._region) - - def includes_ofs(self, ofs): - """:return: True if the given absolute offset is contained in the cursors - current region - - **Note:** cursor must be valid for this to work""" - # unroll methods - return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) - - def file_size(self): - """:return: size of the underlying file""" - return self._rlist.file_size() - - def path_or_fd(self): - """:return: path or file decriptor of the underlying mapped file""" - return self._rlist.path_or_fd() + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested when calling use_region() + + **Note:** You can only obtain a buffer if this instance is_valid() ! + + **Note:** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" + return buffer(self._region.buffer(), self._ofs, self._size) + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor + + **Note:** only if is_valid() is True""" + return self._region._b + self._ofs + + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region._b + self._ofs + self._size + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region_ref(self): + """:return: weak ref to our mapped region. + :raise AssertionError: if we have no current region. This is only useful for debugging""" + if self._region is None: + raise AssertionError("region not set") + return ref(self._region) + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + + **Note:** cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path_or_fd(self): + """:return: path or file decriptor of the underlying mapped file""" + return self._rlist.path_or_fd() - def path(self): - """:return: path of the underlying mapped file - :raise ValueError: if attached path is not a path""" - if isinstance(self._rlist.path_or_fd(), int): - raise ValueError("Path queried although mapping was applied to a file descriptor") - # END handle type - return self._rlist.path_or_fd() - - def fd(self): - """:return: file descriptor used to create the underlying mapping. - - **Note:** it is not required to be valid anymore - :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), basestring): - raise ValueError("File descriptor queried although mapping was generated from path") - #END handle type - return self._rlist.path_or_fd() - - #} END interface - - + def path(self): + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + + **Note:** it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), basestring): + raise ValueError("File descriptor queried although mapping was generated from path") + #END handle type + return self._rlist.path_or_fd() + + #} END interface + + class StaticWindowMapManager(object): - """Provides a manager which will produce single size cursors that are allowed - to always map the whole file. - - Clients must be written to specifically know that they are accessing their data - through a StaticWindowMapManager, as they otherwise have to deal with their window size. - - These clients would have to use a SlidingWindowMapBuffer to hide this fact. - - This type will always use a maximum window size, and optimize certain methods to - acomodate this fact""" - - __slots__ = [ - '_fdict', # mapping of path -> StorageHelper (of some kind - '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount ofmemory we may allocate - '_max_handle_count', # maximum amount of handles to keep open - '_memory_size', # currently allocated memory size - '_handle_count', # amount of currently allocated file handles - ] - - #{ Configuration - MapRegionListCls = MapRegionList - MapWindowCls = MapWindow - MapRegionCls = MapRegion - WindowCursorCls = WindowCursor - #} END configuration - - _MB_in_bytes = 1024 * 1024 - - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): - """initialize the manager with the given parameters. - :param window_size: if -1, a default window size will be chosen depending on - the operating system's architechture. It will internally be quantified to a multiple of the page size - If 0, the window may have any size, which basically results in mapping the whole file at one - :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. - If 0, a viable default iwll be set dependning on the system's architecture. - It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate - :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, - the manager will free as many handles as posisble""" - self._fdict = dict() - self._window_size = window_size - self._max_memory_size = max_memory_size - self._max_handle_count = max_open_handles - self._memory_size = 0 - self._handle_count = 0 - - if window_size < 0: - coeff = 32 - if is_64_bit(): - coeff = 1024 - #END handle arch - self._window_size = coeff * self._MB_in_bytes - # END handle max window size - - if max_memory_size == 0: - coeff = 512 - if is_64_bit(): - coeff = 8192 - #END handle arch - self._max_memory_size = coeff * self._MB_in_bytes - #END handle max memory size - - #{ Internal Methods - - def _collect_lru_region(self, size): - """Unmap the region which was least-recently used and has no client - :param size: size of the region we want to map next (assuming its not already mapped partially or full - if 0, we try to free any available region - :return: Amount of freed regions - - **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. - - **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" - num_found = 0 - while (size == 0) or (self._memory_size + size > self._max_memory_size): - lru_region = None - lru_list = None - for regions in self._fdict.itervalues(): - for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): - lru_region = region - lru_list = regions - # END update lru_region - #END for each region - #END for each regions list - - if lru_region is None: - break - #END handle region not found - - num_found += 1 - del(lru_list[lru_list.index(lru_region)]) - self._memory_size -= lru_region.size() - self._handle_count -= 1 - #END while there is more memory to free - return num_found - - def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, - see MapCursor.use_region. - :param a: A regions (a)rray - :return: The newly created region""" - if self._memory_size + size > self._max_memory_size: - self._collect_lru_region(size) - #END handle collection - - r = None - if a: - assert len(a) == 1 - r = a[0] - else: - try: - r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions - - self._handle_count += 1 - self._memory_size += r.size() - a.append(r) - # END handle array - - assert r.includes_ofs(offset) - return r + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. + + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. + + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + acomodate this fact""" + + __slots__ = [ + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + #{ Configuration + MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion + WindowCursorCls = WindowCursor + #} END configuration + + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + """initialize the manager with the given parameters. + :param window_size: if -1, a default window size will be chosen depending on + the operating system's architechture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default iwll be set dependning on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate + :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, + the manager will free as many handles as posisble""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size < 0: + coeff = 32 + if is_64_bit(): + coeff = 1024 + #END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 512 + if is_64_bit(): + coeff = 8192 + #END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + #END handle max memory size + + #{ Internal Methods + + def _collect_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :return: Amount of freed regions + + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): + lru_region = region + lru_list = regions + # END update lru_region + #END for each region + #END for each regions list + + if lru_region is None: + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + #END while there is more memory to free + return num_found + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) + #END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.append(r) + # END handle array + + assert r.includes_ofs(offset) + return r - #}END internal methods - - #{ Interface - def make_cursor(self, path_or_fd): - """ - :return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory - - **Note:** if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - - **Note:** file descriptors are problematic as they are not necessarily unique, as two - different files opened and closed in succession might have the same file descriptor id. - - **Note:** Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" - regions = self._fdict.get(path_or_fd) - if regions is None: - regions = self.MapRegionListCls(path_or_fd) - self._fdict[path_or_fd] = regions - # END obtain region for path - return self.WindowCursorCls(self, regions) - - def collect(self): - """Collect all available free-to-collect mapped regions - :return: Amount of freed handles""" - return self._collect_lru_region(0) - - def num_file_handles(self): - """:return: amount of file handles in use. Each mapped region uses one file handle""" - return self._handle_count - - def num_open_files(self): - """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) - - def window_size(self): - """:return: size of each window when allocating new regions""" - return self._window_size - - def mapped_memory_size(self): - """:return: amount of bytes currently mapped in total""" - return self._memory_size - - def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" - return self._max_handle_count - - def max_mapped_memory_size(self): - """:return: maximum amount of memory we may allocate""" - return self._max_memory_size - - #} END interface - - #{ Special Purpose Interface - - def force_map_handle_removal_win(self, base_path): - """ONLY AVAILABLE ON WINDOWS - On windows removing files is not allowed if anybody still has it opened. - If this process is ourselves, and if the whole process uses this memory - manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to - allow the respective operation after all. - The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep - the cursors alive will not be using it anymore. They need to be recreated ! - :return: Amount of closed handles - - **Note:** does nothing on non-windows platforms""" - if sys.platform != 'win32': - return - #END early bailout - - num_closed = 0 - for path, rlist in self._fdict.iteritems(): - if path.startswith(base_path): - for region in rlist: - region._mf.close() - num_closed += 1 - #END path matches - #END for each path - return num_closed - #} END special purpose interface - - - + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.WindowCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + + **Note:** does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface + + + class SlidingWindowMapManager(StaticWindowMapManager): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily - obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles - which result from each mmap call, the least recently used, and currently unused mapped regions - are unloaded automatically. - - **Note:** currently not thread-safe ! - - **Note:** in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than - a safe amount of memory already, which would possibly cause memory allocations to fail as our address - space is full.""" - - __slots__ = tuple() - - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): - """Adjusts the default window size to -1""" - super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - - def _obtain_region(self, a, offset, size, flags, is_recursive): - # bisect to find an existing region. The c++ implementation cannot - # do that as it uses a linked list for regions. - r = None - lo = 0 - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - ofs = a[mid]._b - if ofs <= offset: - if a[mid].includes_ofs(offset): - r = a[mid] - break - #END have region - lo = mid+1 - else: - hi = mid - #END handle position - #END while bisecting - - if r is None: - window_size = self._window_size - left = self.MapWindowCls(0, 0) - mid = self.MapWindowCls(offset, size) - right = self.MapWindowCls(a.file_size(), 0) - - # we want to honor the max memory size, and assure we have anough - # memory available - # Save calls ! - if self._memory_size + window_size > self._max_memory_size: - self._collect_lru_region(window_size) - #END handle collection - - # we assume the list remains sorted by offset - insert_pos = 0 - len_regions = len(a) - if len_regions == 1: - if a[0]._b <= offset: - insert_pos = 1 - #END maintain sort - else: - # find insert position - insert_pos = len_regions - for i, region in enumerate(a): - if region._b > offset: - insert_pos = i - break - #END if insert position is correct - #END for each region - # END obtain insert pos - - # adjust the actual offset and size values to create the largest - # possible mapping - if insert_pos == 0: - if len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side - else: - if insert_pos != len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - # END adjust right window - left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows - - mid.extend_left_to(left, window_size) - mid.extend_right_to(right, window_size) - mid.align() - - # it can happen that we align beyond the end of the file - if mid.ofs_end() > right.ofs: - mid.size = right.ofs - mid.ofs - #END readjust size - - # insert new region at the right offset to keep the order - try: - if self._handle_count >= self._max_handle_count: - raise Exception - #END assert own imposed max file handles - r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions - - self._handle_count += 1 - self._memory_size += r.size() - a.insert(insert_pos, r) - # END create new region - return r - - + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to -1""" + super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + + def _obtain_region(self, a, offset, size, flags, is_recursive): + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + #END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r + + diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 6957dcab0..21e6c5a09 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -9,57 +9,57 @@ #{ Utilities class FileCreator(object): - """A instance which creates a temporary file with a prefix and a given size - and provides this info to the user. - Once it gets deleted, it will remove the temporary file as well.""" - __slots__ = ("_size", "_path") - - def __init__(self, size, prefix=''): - assert size, "Require size to be larger 0" - - self._path = tempfile.mktemp(prefix=prefix) - self._size = size - - fp = open(self._path, "wb") - fp.seek(size-1) - fp.write('1') - fp.close() - - assert os.path.getsize(self.path) == size + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._path = tempfile.mktemp(prefix=prefix) + self._size = size + + fp = open(self._path, "wb") + fp.seek(size-1) + fp.write('1') + fp.close() + + assert os.path.getsize(self.path) == size - def __del__(self): - try: - os.remove(self.path) - except OSError: - pass - #END exception handling - + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + #END exception handling + - @property - def path(self): - return self._path - - @property - def size(self): - return self._size + @property + def path(self): + return self._path + + @property + def size(self): + return self._size #} END utilities class TestBase(TestCase): - """Foundation used by all tests""" - - #{ Configuration - k_window_test_size = 1000 * 1000 * 8 + 5195 - #} END configuration - - #{ Overrides - @classmethod - def setUpAll(cls): - # nothing for now - pass - - #END overrides - - #{ Interface - - #} END interface + """Foundation used by all tests""" + + #{ Configuration + k_window_test_size = 1000 * 1000 * 8 + 5195 + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + #END overrides + + #{ Interface + + #} END interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 9881c6294..4bdcb76f5 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -10,108 +10,108 @@ man_optimal = SlidingWindowMapManager() -man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, - max_memory_size=TestBase.k_window_test_size/3, - max_open_handles=15) +man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, + max_memory_size=TestBase.k_window_test_size/3, + max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): - - def test_basics(self): - fc = FileCreator(self.k_window_test_size, "buffer_test") - - # invalid paths fail upon construction - c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - - buf = SlidingWindowMapBuffer() # can create uninitailized buffers - assert buf.cursor() is None - - # can call end access any time - buf.end_access() - buf.end_access() - assert len(buf) == 0 - - # begin access can revive it, if the offset is suitable - offset = 100 - assert buf.begin_access(c, fc.size) == False - assert buf.begin_access(c, offset) == True - assert len(buf) == fc.size - offset - assert buf.cursor().is_valid() - - # empty begin access keeps it valid on the same path, but alters the offset - assert buf.begin_access() == True - assert len(buf) == fc.size - assert buf.cursor().is_valid() - - # simple access - data = open(fc.path, 'rb').read() - assert data[offset] == buf[0] - assert data[offset:offset*2] == buf[0:offset] - - # negative indices, partial slices - assert buf[-1] == buf[len(buf)-1] - assert buf[-10:] == buf[len(buf)-10:len(buf)] - - # end access makes its cursor invalid - buf.end_access() - assert not buf.cursor().is_valid() - assert buf.cursor().is_associated() # but it remains associated - - # an empty begin access fixes it up again - assert buf.begin_access() == True and buf.cursor().is_valid() - del(buf) # ends access automatically - del(c) - - assert man_optimal.num_file_handles() == 1 - - # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to - # exagerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which - # will produce small mappings only ! - max_num_accesses = 100 - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case'), - (static_man, 'static optimial')): - buf = SlidingWindowMapBuffer(manager.make_cursor(item)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - #END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000*1000) - mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) - # END handle access mode - # END for each manager - # END for each input - os.close(fd) + + def test_basics(self): + fc = FileCreator(self.k_window_test_size, "buffer_test") + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitailized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + data = open(fc.path, 'rb').read() + assert data[offset] == buf[0] + assert data[offset:offset*2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf)-1] + assert buf[-10:] == buf[len(buf)-10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with rnadom access and a full mapping - we don't want to + # exagerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimial')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 27be686ad..46429a419 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -12,203 +12,203 @@ from copy import copy class TestMMan(TestBase): - - def test_cursor(self): - fc = FileCreator(self.k_window_test_size, "cursor_test") - - man = SlidingWindowMapManager() - ci = WindowCursor(man) # invalid cursor - assert not ci.is_valid() - assert not ci.is_associated() - assert ci.size() == 0 # this is cached, so we can query it in invalid state - - cv = man.make_cursor(fc.path) - assert not cv.is_valid() # no region mapped yet - assert cv.is_associated()# but it know where to map it from - assert cv.file_size() == fc.size - assert cv.path() == fc.path - - # copy module - cio = copy(cv) - assert not cio.is_valid() and cio.is_associated() - - # assign method - assert not ci.is_associated() - ci.assign(cv) - assert not ci.is_valid() and ci.is_associated() - - # unuse non-existing region is fine - cv.unuse_region() - cv.unuse_region() - - # destruction is fine (even multiple times) - cv._destroy() - WindowCursor(man)._destroy() - - def test_memory_manager(self): - slide_man = SlidingWindowMapManager() - static_man = StaticWindowMapManager() - - for man in (static_man, slide_man): - assert man.num_file_handles() == 0 - assert man.num_open_files() == 0 - winsize_cmp_val = 0 - if isinstance(man, StaticWindowMapManager): - winsize_cmp_val = -1 - #END handle window size - assert man.window_size() > winsize_cmp_val - assert man.mapped_memory_size() == 0 - assert man.max_mapped_memory_size() > 0 - - # collection doesn't raise in 'any' mode - man._collect_lru_region(0) - # doesn't raise if we are within the limit - man._collect_lru_region(10) - - # doesn't fail if we overallocate - assert man._collect_lru_region(sys.maxint) == 0 - - # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] - - if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) - else: - self.failUnlessRaises(ValueError, c.fd) - #END handle value error - #END for each input - os.close(fd) - # END for each manager type - - def test_memman_operation(self): - # test more access, force it to actually unmap regions - fc = FileCreator(self.k_window_test_size, "manager_operation_test") - data = open(fc.path, 'rb').read() - fd = os.open(fc.path, os.O_RDONLY) - max_num_handles = 15 - #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - - #assert c.size() == size # the cursor may overallocate in its static version - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - if man.window_size(): - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) - else: - assert rr().size() == fc.size - #END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - - # obtain second window, which spans the first part of the file - it is a still the same window - nsize = (size or fc.size) - 10 - assert c.use_region(0, nsize).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == nsize - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:nsize] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - (size or c.size()) + overshoot - assert c.use_region(base_offset, size).is_valid() - if man.window_size(): - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - else: - assert c.size() < fc.size - #END ignore static managers which only have one handle per file - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - #END ignore this for static managers - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) - - # precondition - if man.window_size(): - assert max_mapped_memory_size >= mapped_memory_size() - #END statics will overshoot, which is fine - assert max_file_handles >= num_file_handles() - assert c.use_region(base_offset, (size or c.size())).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize - - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - #END for each item - # END for each manager type - os.close(fd) + + def test_cursor(self): + fc = FileCreator(self.k_window_test_size, "cursor_test") + + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated()# but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path + + # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() + + # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + + # destruction is fine (even multiple times) + cv._destroy() + WindowCursor(man)._destroy() + + def test_memory_manager(self): + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() + + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + #END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + + # doesn't fail if we overallocate + assert man._collect_lru_region(sys.maxint) == 0 + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error + #END for each input + os.close(fd) + # END for each manager type + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + fc = FileCreator(self.k_window_test_size, "manager_operation_test") + data = open(fc.path, 'rb').read() + fd = os.open(fc.path, os.O_RDONLY) + max_num_handles = 15 + #small_size = + for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() / 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + + #assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + else: + assert rr().size() == fc.size + #END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + #END ignore static managers which only have one handle per file + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + #END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + #END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + # END for each manager type + os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index a9f4b1c08..4e1a5764b 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,83 +1,83 @@ from lib import TestBase class TestTutorial(TestBase): - - def test_example(self): - # Memory Managers - ################## - import smmap - # This instance should be globally available in your application - # It is configured to be well suitable for 32-bit or 64 bit applications. - mman = smmap.SlidingWindowMapManager() - - # the manager provides much useful information about its current state - # like the amount of open file handles or the amount of mapped memory - assert mman.num_file_handles() == 0 - assert mman.mapped_memory_size() == 0 - # and many more ... - - # Cursors - ########## - import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") - - # obtain a cursor to access some file. - c = mman.make_cursor(fc.path) - - # the cursor is now associated with the file, but not yet usable - assert c.is_associated() - assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to - # access. The following just says you want as much data as possible starting - # from offset 0. - # To be sure your region could be mapped, query for validity - assert c.use_region().is_valid() # use_region returns self - - # once a region was mapped, you must query its dimension regularly - # to assure you don't try to access its buffer out of its bounds - assert c.size() - c.buffer()[0] # first byte - c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size()-1] # last byte - - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - - # you can query absolute offsets, and check whether an offset is included - # in the cursor's data. - assert c.ofs_begin() < c.ofs_end() - assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the - # cursor will be come invalid. It cannot be used in that state - assert not c.use_region(fc.size, 100).is_valid() - # map as much as possible after skipping the first 100 bytes - assert c.use_region(100).is_valid() - - # You can explicitly free cursor resources by unusing the cursor's region - c.unuse_region() - assert not c.is_valid() - - # Buffers - ######### - # Create a default buffer which can operate on the whole file - buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - - # you can use it right away - assert buf.cursor().is_valid() - - buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:]# access the last ten bytes - - # If you want to keep the instance between different accesses, use the - # dedicated methods - buf.end_access() - assert not buf.cursor().is_valid() # you cannot use the buffer anymore - assert buf.begin_access(offset=10) # start using the buffer at an offset - - # it will stop using resources automatically once it goes out of scope - + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 096c5f6df..2df0660be 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -6,109 +6,109 @@ import sys class TestMMan(TestBase): - - def test_window(self): - wl = MapWindow(0, 1) # left - wc = MapWindow(1, 1) # center - wc2 = MapWindow(10, 5) # another center - wr = MapWindow(8000, 50) # right - - assert wl.ofs_end() == 1 - assert wc.ofs_end() == 2 - assert wr.ofs_end() == 8050 - - # extension does nothing if already in place - maxsize = 100 - wc.extend_left_to(wl, maxsize) - assert wc.ofs == 1 and wc.size == 1 - wl.extend_right_to(wc, maxsize) - wl.extend_right_to(wc, maxsize) - assert wl.ofs == 0 and wl.size == 1 - - # an actual left extension - pofs_end = wc2.ofs_end() - wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - - # respects maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - - # without maxsize - wc.extend_right_to(wr, sys.maxint) - assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - - # extend left - wr.extend_left_to(wc2, maxsize) - wr.extend_left_to(wc2, maxsize) - assert wr.size == maxsize - - wr.extend_left_to(wc2, sys.maxint) - assert wr.ofs == wc2.ofs_end() - - wc.align() - assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) - - def test_region(self): - fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size / 2 - rofs = align_to_mmap(4200, False) - rfull = MapRegion(fc.path, 0, fc.size) - rhalfofs = MapRegion(fc.path, rofs, fc.size) - rhalfsize = MapRegion(fc.path, 0, half_size) - - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always - - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - # with the values we have, this test only works on windows where an alignment - # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions - # as they use different mapping techniques to circumvent the missing offset - # argument of mmap. - if sys.platform != 'win32': - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - #END handle platforms - - # auto-refcount - assert rfull.client_count() == 1 - rfull2 = rfull - assert rfull.client_count() == 2 - - # usage - assert rfull.usage_count() == 0 - rfull.increment_usage_count() - assert rfull.usage_count() == 1 - - # window constructor - w = MapWindow.from_region(rfull) - assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - - def test_region_list(self): - fc = FileCreator(100, "sample_file") - - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - ml = MapRegionList(item) - - assert ml.client_count() == 1 - - assert len(ml) == 0 - assert ml.path_or_fd() == item - assert ml.file_size() == fc.size - #END handle input - os.close(fd) - - def test_util(self): - assert isinstance(is_64_bit(), bool) # just call it - assert align_to_mmap(1, False) == 0 - assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY - + + def test_window(self): + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) + + def test_region(self): + fc = FileCreator(self.k_window_test_size, "window_test") + half_size = fc.size / 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + # with the values we have, this test only works on windows where an alignment + # size of 4096 is assumed. + # We only test on linux as it is inconsitent between the python versions + # as they use different mapping techniques to circumvent the missing offset + # argument of mmap. + if sys.platform != 'win32': + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + #END handle platforms + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + + # usage + assert rfull.usage_count() == 0 + rfull.increment_usage_count() + assert rfull.usage_count() == 1 + + # window constructor + w = MapWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + fc = FileCreator(100, "sample_file") + + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert ml.client_count() == 1 + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + #END handle input + os.close(fd) + + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY + diff --git a/smmap/util.py b/smmap/util.py index b0fd83b3f..c6710b3fe 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -5,36 +5,36 @@ from mmap import mmap, ACCESS_READ try: - from mmap import ALLOCATIONGRANULARITY + from mmap import ALLOCATIONGRANULARITY except ImportError: - # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly - # useful for aligning the offset. The offset argument doesn't exist there though - from mmap import PAGESIZE as ALLOCATIONGRANULARITY + # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly + # useful for aligning the offset. The offset argument doesn't exist there though + from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance from sys import getrefcount -__all__ = [ "align_to_mmap", "is_64_bit", - "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] +__all__ = [ "align_to_mmap", "is_64_bit", + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities def align_to_mmap(num, round_up): - """ - Align the given integer number to the closest page offset, which usually is 4096 bytes. - - :param round_up: if True, the next higher multiple of page size is used, otherwise - the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) - :return: num rounded to closest page""" - res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; - if round_up and (res != num): - res += ALLOCATIONGRANULARITY - #END handle size - return res; - + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + if round_up and (res != num): + res += ALLOCATIONGRANULARITY + #END handle size + return res; + def is_64_bit(): - """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxint > (1<<32) - 1 + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxint > (1<<32) - 1 #}END utilities @@ -42,228 +42,228 @@ def is_64_bit(): #{ Utility Classes class MapWindow(object): - """Utility type which is used to snap windows towards each other, and to adjust their size""" - __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes - ) + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) - def __init__(self, offset, size): - self.ofs = offset - self.size = size + def __init__(self, offset, size): + self.ofs = offset + self.size = size - def __repr__(self): - return "MapWindow(%i, %i)" % (self.ofs, self.size) + def __repr__(self): + return "MapWindow(%i, %i)" % (self.ofs, self.size) - @classmethod - def from_region(cls, region): - """:return: new window from a region""" - return cls(region._b, region.size()) + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region._b, region.size()) - def ofs_end(self): - return self.ofs + self.size + def ofs_end(self): + return self.ofs + self.size - def align(self): - """Assures the previous window area is contained in the new one""" - nofs = align_to_mmap(self.ofs, 0) - self.size += self.ofs - nofs # keep size constant - self.ofs = nofs - self.size = align_to_mmap(self.size, 1) + def align(self): + """Assures the previous window area is contained in the new one""" + nofs = align_to_mmap(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs + self.size = align_to_mmap(self.size, 1) - def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, - but don't make yourself larger than max_size. - The resize will assure that the new window still contains the old window area""" - rofs = self.ofs - window.ofs_end() - nsize = rofs + self.size - rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs - self.size += rofs + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs - def extend_right_to(self, window, max_size): - """Adjust the size to make our window end where the right window begins, but don't - get larger than max_size""" - self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) class MapRegion(object): - """Defines a mapped region of memory, aligned to pagesizes - - **Note:** deallocates used region automatically on destruction""" - __slots__ = [ - '_b' , # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_size', # cached size of our memory map - '__weakref__' - ] - _need_compat_layer = sys.version_info[1] < 6 - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot - - #{ Configuration - # Used for testing only. If True, all data will be loaded into memory at once. - # This makes sure no file handles will remain open. - _test_read_into_memory = False - #} END configuration - - - def __init__(self, path_or_fd, ofs, size, flags = 0): - """Initialize a region, allocate the memory map - :param path_or_fd: path to the file to map, or the opened file descriptor - :param ofs: **aligned** offset into the file to be mapped - :param size: if size is larger then the file on disk, the whole file will be - allocated the the size automatically adjusted - :param flags: additional flags to be given when opening the file. - :raise Exception: if no memory can be allocated""" - self._b = ofs - self._size = 0 - self._uc = 0 - - if isinstance(path_or_fd, int): - fd = path_or_fd - else: - fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) - #END handle fd - - try: - kwargs = dict(access=ACCESS_READ, offset=ofs) - corrected_size = size - sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will - # bark that the size is too large ... many extra file accesses because - # if this ... argh ! - actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) - if self._test_read_into_memory: - self._mf = self._read_into_memory(fd, ofs, actual_size) - else: - self._mf = mmap(fd, actual_size, **kwargs) - #END handle memory mode - - self._size = len(self._mf) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, self._size) - #END handle buffer wrapping - finally: - if isinstance(path_or_fd, basestring): - os.close(fd) - #END only close it if we opened it - #END close file handle - - def _read_into_memory(self, fd, offset, size): - """:return: string data as read from the given file descriptor, offset and size """ - os.lseek(fd, offset, os.SEEK_SET) - mf = '' - bytes_todo = size - while bytes_todo: - chunk = 1024*1024 - d = os.read(fd, chunk) - bytes_todo -= len(d) - mf += d - #END loop copy items - return mf - - def __repr__(self): - return "MapRegion<%i, %i>" % (self._b, self.size()) - - #{ Interface + """Defines a mapped region of memory, aligned to pagesizes + + **Note:** deallocates used region automatically on destruction""" + __slots__ = [ + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] + _need_compat_layer = sys.version_info[1] < 6 + + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + + #{ Configuration + # Used for testing only. If True, all data will be loaded into memory at once. + # This makes sure no file handles will remain open. + _test_read_into_memory = False + #} END configuration + + + def __init__(self, path_or_fd, ofs, size, flags = 0): + """Initialize a region, allocate the memory map + :param path_or_fd: path to the file to map, or the opened file descriptor + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._size = 0 + self._uc = 0 + + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + #END handle fd + + try: + kwargs = dict(access=ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + sizeofs = 0 + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + if self._test_read_into_memory: + self._mf = self._read_into_memory(fd, ofs, actual_size) + else: + self._mf = mmap(fd, actual_size, **kwargs) + #END handle memory mode + + self._size = len(self._mf) + + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, self._size) + #END handle buffer wrapping + finally: + if isinstance(path_or_fd, basestring): + os.close(fd) + #END only close it if we opened it + #END close file handle + + def _read_into_memory(self, fd, offset, size): + """:return: string data as read from the given file descriptor, offset and size """ + os.lseek(fd, offset, os.SEEK_SET) + mf = '' + bytes_todo = size + while bytes_todo: + chunk = 1024*1024 + d = os.read(fd, chunk) + bytes_todo -= len(d) + mf += d + #END loop copy items + return mf + + def __repr__(self): + return "MapRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface - def buffer(self): - """:return: a buffer containing the memory""" - return self._mf - - def map(self): - """:return: a memory map containing the memory""" - return self._mf - - def ofs_begin(self): - """:return: absolute byte offset to the first byte of the mapping""" - return self._b - - def size(self): - """:return: total size of the mapped region in bytes""" - return self._size - - def ofs_end(self): - """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self._size - - def includes_ofs(self, ofs): - """:return: True if the given offset can be read in our mapped region""" - return self._b <= ofs < self._b + self._size - - def client_count(self): - """:return: number of clients currently using this region""" - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 - - def usage_count(self): - """:return: amount of usages so far""" - return self._uc - - def increment_usage_count(self): - """Adjust the usage count by the given positive or negative offset""" - self._uc += 1 - - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return self._size - self._b - - def ofs_end(self): - # always the size - we are as large as it gets - return self._size - - def buffer(self): - return self._mfb - - def includes_ofs(self, ofs): - return self._b <= ofs < self._size - #END handle compat layer - - #} END interface - - + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + + def map(self): + """:return: a memory map containing the memory""" + return self._mf + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return self._size + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self._size + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return self._b <= ofs < self._b + self._size + + def client_count(self): + """:return: number of clients currently using this region""" + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def increment_usage_count(self): + """Adjust the usage count by the given positive or negative offset""" + self._uc += 1 + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return self._size - self._b + + def ofs_end(self): + # always the size - we are as large as it gets + return self._size + + def buffer(self): + return self._mfb + + def includes_ofs(self, ofs): + return self._b <= ofs < self._size + #END handle compat layer + + #} END interface + + class MapRegionList(list): - """List of MapRegion instances associating a path with a list of regions.""" - __slots__ = ( - '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map - ) - - def __new__(cls, path): - return super(MapRegionList, cls).__new__(cls) - - def __init__(self, path_or_fd): - self._path_or_fd = path_or_fd - self._file_size = None - - def client_count(self): - """:return: amount of clients which hold a reference to this instance""" - return getrefcount(self)-3 - - def path_or_fd(self): - """:return: path or file descriptor we are attached to""" - return self._path_or_fd - - def file_size(self): - """:return: size of file we manager""" - if self._file_size is None: - if isinstance(self._path_or_fd, basestring): - self._file_size = os.stat(self._path_or_fd).st_size - else: - self._file_size = os.fstat(self._path_or_fd).st_size - #END handle path type - #END update file size - return self._file_size - + """List of MapRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MapRegionList, cls).__new__(cls) + + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd + self._file_size = None + + def client_count(self): + """:return: amount of clients which hold a reference to this instance""" + return getrefcount(self)-3 + + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + if isinstance(self._path_or_fd, basestring): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + #END handle path type + #END update file size + return self._file_size + #} END utilty classes From 6576d5503a64d124fd7bcf639cc8955918b3ac43 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Feb 2014 20:51:43 +0100 Subject: [PATCH 0217/3719] tabs to spaces --- gitdb/__init__.py | 22 +- gitdb/base.py | 556 ++--- gitdb/db/base.py | 586 ++--- gitdb/db/git.py | 134 +- gitdb/db/loose.py | 470 ++-- gitdb/db/mem.py | 188 +- gitdb/db/pack.py | 374 ++-- gitdb/db/ref.py | 138 +- gitdb/exc.py | 30 +- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 1206 +++++------ gitdb/pack.py | 1920 ++++++++--------- gitdb/stream.py | 1288 +++++------ gitdb/test/__init__.py | 8 +- gitdb/test/db/lib.py | 376 ++-- gitdb/test/db/test_git.py | 74 +- gitdb/test/db/test_loose.py | 50 +- gitdb/test/db/test_mem.py | 46 +- gitdb/test/db/test_pack.py | 118 +- gitdb/test/db/test_ref.py | 98 +- gitdb/test/lib.py | 206 +- gitdb/test/performance/lib.py | 56 +- gitdb/test/performance/test_pack.py | 150 +- gitdb/test/performance/test_pack_streaming.py | 120 +- gitdb/test/performance/test_stream.py | 324 +-- gitdb/test/test_base.py | 168 +- gitdb/test/test_example.py | 102 +- gitdb/test/test_pack.py | 438 ++-- gitdb/test/test_stream.py | 266 +-- gitdb/test/test_util.py | 184 +- gitdb/util.py | 538 ++--- 32 files changed, 5119 insertions(+), 5119 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 800b292da..ff750d14c 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -9,17 +9,17 @@ #{ Initialization def _init_externals(): - """Initialize external projects by putting them into the path""" - for module in ('async', 'smmap'): - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - - try: - __import__(module) - except ImportError: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) - #END verify import - #END handel imports - + """Initialize external projects by putting them into the path""" + for module in ('async', 'smmap'): + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + + try: + __import__(module) + except ImportError: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + #END verify import + #END handel imports + #} END initialization _init_externals() diff --git a/gitdb/base.py b/gitdb/base.py index ff1062bf6..bad5f7472 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -4,308 +4,308 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( - bin_to_hex, - zlib - ) + bin_to_hex, + zlib + ) from fun import ( - type_id_to_type_map, - type_to_type_id_map - ) + type_id_to_type_map, + type_to_type_id_map + ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream' ) #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provding information - about the binary sha of the object, the type_string as well as the uncompressed size - in bytes. - - It can be accessed using tuple notation and using attribute access notation:: - - assert dbi[0] == dbi.binsha - assert dbi[1] == dbi.type - assert dbi[2] == dbi.size - - The type is designed to be as lighteight as possible.""" - __slots__ = tuple() - - def __new__(cls, sha, type, size): - return tuple.__new__(cls, (sha, type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return self[0] - - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return bin_to_hex(self[0]) - - @property - def type(self): - return self[1] - - @property - def type_id(self): - return type_to_type_id_map[self[1]] - - @property - def size(self): - return self[2] - #} END interface - - + """Carries information about an object in an ODB, provding information + about the binary sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.binsha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return self[0] + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + @property + def type(self): + return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] + + @property + def size(self): + return self[2] + #} END interface + + class OPackInfo(tuple): - """As OInfo, but provides a type_id property to retrieve the numerical type id, and - does not include a sha. - - Additionally, the pack_offset is the absolute offset into the packfile at which - all object information is located. The data_offset property points to the abosolute - location in the pack at which that actual data stream can be found.""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size): - return tuple.__new__(cls, (packoffset,type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - - @property - def pack_offset(self): - return self[0] - - @property - def type(self): - return type_id_to_type_map[self[1]] - - @property - def type_id(self): - return self[1] - - @property - def size(self): - return self[2] - - #} END interface - - + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the abosolute + location in the pack at which that actual data stream can be found.""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset,type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def pack_offset(self): + return self[0] + + @property + def type(self): + return type_id_to_type_map[self[1]] + + @property + def type_id(self): + return self[1] + + @property + def size(self): + return self[2] + + #} END interface + + class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, - or the negative offset from the pack_offset, so that pack_offset - delta_info yields - the pack offset of the base object""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, delta_info): - return tuple.__new__(cls, (packoffset, type, size, delta_info)) - - #{ Interface - @property - def delta_info(self): - return self[3] - #} END interface - - + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional - information about the stream. - Generally, ODB streams are read-only as objects are immutable""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - - def __init__(self, *args, **kwargs): - tuple.__init__(self) - - #{ Stream Reader Interface - - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - - #} END stream reader interface - - + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + + #} END stream reader interface + + class ODeltaStream(OStream): - """Uses size info of its stream, delaying reads""" - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - #{ Stream Reader Interface - - @property - def size(self): - return self[3].size - - #} END stream reader interface - - + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + + #} END stream reader interface + + class OPackStream(OPackInfo): - """Next to pack object information, a stream outputting an undeltified base object - is provided""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, stream, *args): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (packoffset, type, size, stream)) - - #{ Stream Reader Interface - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (packoffset, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface - + class ODeltaPackStream(ODeltaPackInfo): - """Provides a stream outputting the uncompressed offset delta information""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, delta_info, stream): - return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface - def read(self, size=-1): - return self[4].read(size) - - @property - def stream(self): - return self[4] - #} END stream reader interface + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] + #} END stream reader interface class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow - the ODB to record information about the operations outcome right in this instance. - - It provides interfaces for the OStream and a StreamReader to allow the instance - to blend in without prior conversion. - - The only method your content stream must support is 'read'""" - __slots__ = tuple() - - def __new__(cls, type, size, stream, sha=None): - return list.__new__(cls, (sha, type, size, stream, None)) - - def __init__(self, type, size, stream, sha=None): - list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return bin_to_hex(self[0]) - - def _error(self): - """:return: the error that occurred when processing the stream, or None""" - return self[4] - - def _set_error(self, exc): - """Set this input stream to the given exc, may be None to reset the error""" - self[4] = exc - - error = property(_error, _set_error) - - #} END interface - - #{ Stream Reader Interface - - def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on - to our internal stream""" - return self[3].read(size) - - #} END stream reader interface - - #{ interface - - def _set_binsha(self, binsha): - self[0] = binsha - - def _binsha(self): - return self[0] - - binsha = property(_binsha, _set_binsha) - - - def _type(self): - return self[1] - - def _set_type(self, type): - self[1] = type - - type = property(_type, _set_type) - - def _size(self): - return self[2] - - def _set_size(self, size): - self[2] = size - - size = property(_size, _set_size) - - def _stream(self): - return self[3] - - def _set_stream(self, stream): - self[3] = stream - - stream = property(_stream, _set_stream) - - #} END odb info interface - + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_binsha(self, binsha): + self[0] = binsha + + def _binsha(self): + return self[0] + + binsha = property(_binsha, _set_binsha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in - the queried database. The exception attribute provides more information about - the cause of the issue""" - __slots__ = tuple() - - def __new__(cls, sha, exc): - return tuple.__new__(cls, (sha, exc)) - - def __init__(self, sha, exc): - tuple.__init__(self, (sha, exc)) - - @property - def binsha(self): - return self[0] - - @property - def hexsha(self): - return bin_to_hex(self[0]) - - @property - def error(self): - """:return: exception instance explaining the failure""" - return self[1] + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def binsha(self): + return self[0] + + @property + def hexsha(self): + return bin_to_hex(self[0]) + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] class InvalidOStream(InvalidOInfo): - """Carries information about an invalid ODB stream""" - __slots__ = tuple() - + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + #} END ODB Bases diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 984acafbf..867e93a81 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,20 +4,20 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, - join, - LazyMixin, - hex_to_bin - ) + pool, + join, + LazyMixin, + hex_to_bin + ) from gitdb.exc import ( - BadObject, - AmbiguousObjectName - ) + BadObject, + AmbiguousObjectName + ) from async import ( - ChannelThreadTask - ) + ChannelThreadTask + ) from itertools import chain @@ -26,301 +26,301 @@ class ObjectDBR(object): - """Defines an interface for object database lookup. - Objects are identified either by their 20 byte bin sha""" - - def __contains__(self, sha): - return self.has_obj - - #{ Query Interface - def has_object(self, sha): - """ - :return: True if the object identified by the given 20 bytes - binary sha is contained in the database""" - raise NotImplementedError("To be implemented in subclass") - - def has_object_async(self, reader): - """Return a reader yielding information about the membership of objects - as identified by shas - :param reader: Reader yielding 20 byte shas. - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate - whether the given sha exists in the database or not""" - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - - def info(self, sha): - """ :return: OInfo instance - :param sha: bytes binary sha - :raise BadObject:""" - raise NotImplementedError("To be implemented in subclass") - - def info_async(self, reader): - """Retrieve information of a multitude of objects asynchronously - :param reader: Channel yielding the sha's of the objects of interest - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" - task = ChannelThreadTask(reader, str(self.info_async), self.info) - return pool.add_task(task) - - def stream(self, sha): - """:return: OStream instance - :param sha: 20 bytes binary sha - :raise BadObject:""" - raise NotImplementedError("To be implemented in subclass") - - def stream_async(self, reader): - """Retrieve the OStream of multiple objects - :param reader: see ``info`` - :param max_threads: see ``ObjectDBW.store`` - :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to - read all OStreams at once. Instead, read them individually using reader.read(x) - where x is small enough.""" - # base implementation just uses the stream method repeatedly - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) - return pool.add_task(task) - - def size(self): - """:return: amount of objects in this database""" - raise NotImplementedError() - - def sha_iter(self): - """Return iterator yielding 20 byte shas for all objects in this data base""" - raise NotImplementedError() - - #} END query interface - - + """Defines an interface for object database lookup. + Objects are identified either by their 20 byte bin sha""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + :return: True if the object identified by the given 20 bytes + binary sha is contained in the database""" + raise NotImplementedError("To be implemented in subclass") + + def has_object_async(self, reader): + """Return a reader yielding information about the membership of objects + as identified by shas + :param reader: Reader yielding 20 byte shas. + :return: async.Reader yielding tuples of (sha, bool) pairs which indicate + whether the given sha exists in the database or not""" + task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) + return pool.add_task(task) + + def info(self, sha): + """ :return: OInfo instance + :param sha: bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info_async(self, reader): + """Retrieve information of a multitude of objects asynchronously + :param reader: Channel yielding the sha's of the objects of interest + :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" + task = ChannelThreadTask(reader, str(self.info_async), self.info) + return pool.add_task(task) + + def stream(self, sha): + """:return: OStream instance + :param sha: 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream_async(self, reader): + """Retrieve the OStream of multiple objects + :param reader: see ``info`` + :param max_threads: see ``ObjectDBW.store`` + :return: async.Reader yielding OStream|InvalidOStream instances in any order + + **Note:** depending on the system configuration, it might not be possible to + read all OStreams at once. Instead, read them individually using reader.read(x) + where x is small enough.""" + # base implementation just uses the stream method repeatedly + task = ChannelThreadTask(reader, str(self.stream_async), self.stream) + return pool.add_task(task) + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() + + #} END query interface + + class ObjectDBW(object): - """Defines an interface to create objects in the database""" - - def __init__(self, *args, **kwargs): - self._ostream = None - - #{ Edit Interface - def set_ostream(self, stream): - """ - Adjusts the stream to which all data should be sent when storing new objects - - :param stream: if not None, the stream to use, if None the default stream - will be used. - :return: previously installed stream, or None if there was no override - :raise TypeError: if the stream doesn't have the supported functionality""" - cstream = self._ostream - self._ostream = stream - return cstream - - def ostream(self): - """ - :return: overridden output stream this instance will write to, or None - if it will write to the default stream""" - return self._ostream - - def store(self, istream): - """ - Create a new object in the database - :return: the input istream object with its sha set to its corresponding value - - :param istream: IStream compatible instance. If its sha is already set - to a value, the object will just be stored in the our database format, - in which case the input stream is expected to be in object format ( header + contents ). - :raise IOError: if data could not be written""" - raise NotImplementedError("To be implemented in subclass") - - def store_async(self, reader): - """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as - they are computed. - - :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, - or its error attribute will be set to the exception informing about the error. - - :param reader: async.Reader yielding IStream instances. - The same instances will be used in the output channel as were received - in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how - many items have actually been produced.""" - # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) - return pool.add_task(task) - - #} END edit interface - + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """ + Adjusts the stream to which all data should be sent when storing new objects + + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """ + :return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """ + Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + def store_async(self, reader): + """ + Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as + they are computed. + + :return: Channel yielding your IStream which served as input, in any order. + The IStreams sha will be set to the sha it received during the process, + or its error attribute will be set to the exception informing about the error. + + :param reader: async.Reader yielding IStream instances. + The same instances will be used in the output channel as were received + in by the Reader. + + **Note:** As some ODB implementations implement this operation atomic, they might + abort the whole operation if one item could not be processed. Hence check how + many items have actually been produced.""" + # base implementation uses store to perform the work + task = ChannelThreadTask(reader, str(self.store_async), self.store) + return pool.add_task(task) + + #} END edit interface + class FileDBBase(object): - """Provides basic facilities to retrieve files of interest, including - caching facilities to help mapping hexsha's to objects""" - - def __init__(self, root_path): - """Initialize this instance to look for its files at the given root path - All subsequent operations will be relative to this path - :raise InvalidDBRoot: - **Note:** The base will not perform any accessablity checking as the base - might not yet be accessible, but become accessible before the first - access.""" - super(FileDBBase, self).__init__() - self._root_path = root_path - - - #{ Interface - def root_path(self): - """:return: path at which this db operates""" - return self._root_path - - def db_path(self, rela_path): - """ - :return: the given relative path relative to our database root, allowing - to pontentially access datafiles""" - return join(self._root_path, rela_path) - #} END interface - + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + **Note:** The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super(FileDBBase, self).__init__() + self._root_path = root_path + + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, rela_path) + #} END interface + class CachingDB(object): - """A database which uses caches to speed-up access""" - - #{ Interface - def update_cache(self, force=False): - """ - Call this method if the underlying data changed to trigger an update - of the internal caching structures. - - :param force: if True, the update must be performed. Otherwise the implementation - may decide not to perform an update if it thinks nothing has changed. - :return: True if an update was performed as something change indeed""" - - # END interface + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """ + Call this method if the underlying data changed to trigger an update + of the internal caching structures. + + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed - and compound databases.""" - if isinstance(database, CompoundDB): - compounds = list() - dbs = database.databases() - output.extend(db for db in dbs if not isinstance(db, CompoundDB)) - for cdb in (db for db in dbs if isinstance(db, CompoundDB)): - _databases_recursive(cdb, output) - else: - output.append(database) - # END handle database type - + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): - """A database which delegates calls to sub-databases. - - Databases are stored in the lazy-loaded _dbs attribute. - Define _set_cache_ to update it with your databases""" - def _set_cache_(self, attr): - if attr == '_dbs': - self._dbs = list() - elif attr == '_db_cache': - self._db_cache = dict() - else: - super(CompoundDB, self)._set_cache_(attr) - - def _db_query(self, sha): - """:return: database containing the given 20 byte sha - :raise BadObject:""" - # most databases use binary representations, prevent converting - # it everytime a database is being queried - try: - return self._db_cache[sha] - except KeyError: - pass - # END first level cache - - for db in self._dbs: - if db.has_object(sha): - self._db_cache[sha] = db - return db - # END for each database - raise BadObject(sha) - - #{ ObjectDBR interface - - def has_object(self, sha): - try: - self._db_query(sha) - return True - except BadObject: - return False - # END handle exceptions - - def info(self, sha): - return self._db_query(sha).info(sha) - - def stream(self, sha): - return self._db_query(sha).stream(sha) + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super(CompoundDB, self)._set_cache_(attr) + + def _db_query(self, sha): + """:return: database containing the given 20 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it everytime a database is being queried + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + + for db in self._dbs: + if db.has_object(sha): + self._db_cache[sha] = db + return db + # END for each database + raise BadObject(sha) + + #{ ObjectDBR interface + + def has_object(self, sha): + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions + + def info(self, sha): + return self._db_query(sha).info(sha) + + def stream(self, sha): + return self._db_query(sha).stream(sha) - def size(self): - """:return: total size of all contained databases""" - return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) - - def sha_iter(self): - return chain(*(db.sha_iter() for db in self._dbs)) - - #} END object DBR Interface - - #{ Interface - - def databases(self): - """:return: tuple of database instances we use for lookups""" - return tuple(self._dbs) + def size(self): + """:return: total size of all contained databases""" + return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) + + def sha_iter(self): + return chain(*(db.sha_iter() for db in self._dbs)) + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) - def update_cache(self, force=False): - # something might have changed, clear everything - self._db_cache.clear() - stat = False - for db in self._dbs: - if isinstance(db, CachingDB): - stat |= db.update_cache(force) - # END if is caching db - # END for each database to update - return stat - - def partial_to_complete_sha_hex(self, partial_hexsha): - """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha - :param partial_hexsha: hexsha with less than 40 byte - :raise AmbiguousObjectName: """ - databases = list() - _databases_recursive(self, databases) - - len_partial_hexsha = len(partial_hexsha) - if len_partial_hexsha % 2 != 0: - partial_binsha = hex_to_bin(partial_hexsha + "0") - else: - partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - - candidate = None - for db in databases: - full_bin_sha = None - try: - if hasattr(db, 'partial_to_complete_sha_hex'): - full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) - else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) - # END handle database type - except BadObject: - continue - # END ignore bad objects - if full_bin_sha: - if candidate and candidate != full_bin_sha: - raise AmbiguousObjectName(partial_hexsha) - candidate = full_bin_sha - # END handle candidate - # END for each db - if not candidate: - raise BadObject(partial_binsha) - return candidate - - #} END interface - + def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface + diff --git a/gitdb/db/git.py b/gitdb/db/git.py index b8fc46aa0..1d6ad0f26 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -3,10 +3,10 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - CompoundDB, - ObjectDBW, - FileDBBase - ) + CompoundDB, + ObjectDBW, + FileDBBase + ) from loose import LooseObjectDB from pack import PackedDB @@ -14,72 +14,72 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) import os __all__ = ('GitDB', ) class GitDB(FileDBBase, ObjectDBW, CompoundDB): - """A git-style object database, which contains all objects in the 'objects' - subdirectory""" - # Configuration - PackDBCls = PackedDB - LooseDBCls = LooseObjectDB - ReferenceDBCls = ReferenceDB - - # Directories - packs_dir = 'pack' - loose_dir = '' - alternates_dir = os.path.join('info', 'alternates') - - def __init__(self, root_path): - """Initialize ourselves on a git objects directory""" - super(GitDB, self).__init__(root_path) - - def _set_cache_(self, attr): - if attr == '_dbs' or attr == '_loose_db': - self._dbs = list() - loose_db = None - for subpath, dbcls in ((self.packs_dir, self.PackDBCls), - (self.loose_dir, self.LooseDBCls), - (self.alternates_dir, self.ReferenceDBCls)): - path = self.db_path(subpath) - if os.path.exists(path): - self._dbs.append(dbcls(path)) - if dbcls is self.LooseDBCls: - loose_db = self._dbs[-1] - # END remember loose db - # END check path exists - # END for each db type - - # should have at least one subdb - if not self._dbs: - raise InvalidDBRoot(self.root_path()) - # END handle error - - # we the first one should have the store method - assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" - - # finally set the value - self._loose_db = loose_db - else: - super(GitDB, self)._set_cache_(attr) - # END handle attrs - - #{ ObjectDBW interface - - def store(self, istream): - return self._loose_db.store(istream) - - def ostream(self): - return self._loose_db.ostream() - - def set_ostream(self, ostream): - return self._loose_db.set_ostream(ostream) - - #} END objectdbw interface - + """A git-style object database, which contains all objects in the 'objects' + subdirectory""" + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB + + # Directories + packs_dir = 'pack' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super(GitDB, self).__init__(root_path) + + def _set_cache_(self, attr): + if attr == '_dbs' or attr == '_loose_db': + self._dbs = list() + loose_db = None + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + else: + super(GitDB, self)._set_cache_(attr) + # END handle attrs + + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + + #} END objectdbw interface + diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 6cd1cefd5..dc0ea0e3b 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -3,53 +3,53 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - FileDBBase, - ObjectDBR, - ObjectDBW - ) + FileDBBase, + ObjectDBR, + ObjectDBW + ) from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) from gitdb.stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - FDStream, - Sha1Writer - ) + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer + ) from gitdb.base import ( - OStream, - OInfo - ) + OStream, + OInfo + ) from gitdb.util import ( - file_contents_ro_filepath, - ENOENT, - hex_to_bin, - bin_to_hex, - exists, - chmod, - isdir, - isfile, - remove, - mkdir, - rename, - dirname, - basename, - join - ) + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isdir, + isfile, + remove, + mkdir, + rename, + dirname, + basename, + join + ) from gitdb.fun import ( - chunk_size, - loose_object_header_info, - write_object, - stream_copy - ) + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) import tempfile import mmap @@ -61,202 +61,202 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): - """A database which operates on loose object files""" - - # CONFIGURATION - # chunks in which data will be copied between streams - stream_chunk_size = chunk_size - - # On windows we need to keep it writable, otherwise it cannot be removed - # either - new_objects_mode = 0444 - if os.name == 'nt': - new_objects_mode = 0644 - - - def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) - self._hexsha_to_file = dict() - # Additional Flags - might be set to 0 after the first failure - # Depending on the root, this might work for some mounts, for others not, which - # is why it is per instance - self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface - def object_path(self, hexsha): - """ - :return: path at which the object with the given hexsha would be stored, - relative to the database root""" - return join(hexsha[:2], hexsha[2:]) - - def readable_db_object_path(self, hexsha): - """ - :return: readable object path to the object identified by hexsha - :raise BadObject: If the object file does not exist""" - try: - return self._hexsha_to_file[hexsha] - except KeyError: - pass - # END ignore cache misses - - # try filesystem - path = self.db_path(self.object_path(hexsha)) - if exists(path): - self._hexsha_to_file[hexsha] = path - return path - # END handle cache - raise BadObject(hexsha) - - def partial_to_complete_sha_hex(self, partial_hexsha): - """:return: 20 byte binary sha1 string which matches the given name uniquely - :param name: hexadecimal partial name - :raise AmbiguousObjectName: - :raise BadObject: """ - candidate = None - for binsha in self.sha_iter(): - if bin_to_hex(binsha).startswith(partial_hexsha): - # it can't ever find the same object twice - if candidate is not None: - raise AmbiguousObjectName(partial_hexsha) - candidate = binsha - # END for each object - if candidate is None: - raise BadObject(partial_hexsha) - return candidate - - #} END interface - - def _map_loose_object(self, sha): - """ - :return: memory map of that file to allow random read access - :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(bin_to_hex(sha))) - try: - return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) - except OSError,e: - if e.errno != ENOENT: - # try again without noatime - try: - return file_contents_ro_filepath(db_path) - except OSError: - raise BadObject(sha) - # didn't work because of our flag, don't try it again - self._fd_open_flags = 0 - else: - raise BadObject(sha) - # END handle error - # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed - - def set_ostream(self, stream): - """:raise TypeError: if the stream does not support the Sha1Writer interface""" - if stream is not None and not isinstance(stream, Sha1Writer): - raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) - - def info(self, sha): - m = self._map_loose_object(sha) - try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) - finally: - m.close() - # END assure release of system resources - - def stream(self, sha): - m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) - return OStream(sha, type, size, stream) - - def has_object(self, sha): - try: - self.readable_db_object_path(bin_to_hex(sha)) - return True - except BadObject: - return False - # END check existance - - def store(self, istream): - """note: The sha we produce will be hex by nature""" - tmp_path = None - writer = self.ostream() - if writer is None: - # open a tmp file to write the data to - fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - - if istream.binsha is None: - writer = FDCompressedSha1Writer(fd) - else: - writer = FDStream(fd) - # END handle direct stream copies - # END handle custom writer - - try: - try: - if istream.binsha is not None: - # copy as much as possible, the actual uncompressed item size might - # be smaller than the compressed version - stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) - else: - # write object with header, we have to make a new one - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) - # END handle direct stream copies - finally: - if tmp_path: - writer.close() - # END assure target stream is closed - except: - if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - - hexsha = None - if istream.binsha: - hexsha = istream.hexsha - else: - hexsha = writer.sha(as_hex=True) - # END handle sha - - if tmp_path: - obj_path = self.db_path(self.object_path(hexsha)) - obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) - # END handle destination directory - # rename onto existing doesn't work on windows - if os.name == 'nt' and isfile(obj_path): - remove(obj_path) - # END handle win322 - rename(tmp_path, obj_path) - - # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rwrr - chmod(obj_path, self.new_objects_mode) - # END handle dry_run - - istream.binsha = hex_to_bin(hexsha) - return istream - - def sha_iter(self): - # find all files which look like an object, extract sha from there - for root, dirs, files in os.walk(self.root_path()): - root_base = basename(root) - if len(root_base) != 2: - continue - - for f in files: - if len(f) != 38: - continue - yield hex_to_bin(root_base + f) - # END for each file - # END for each walk iteration - - def size(self): - return len(tuple(self.sha_iter())) - + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = 0444 + if os.name == 'nt': + new_objects_mode = 0644 + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(partial_hexsha): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(bin_to_hex(sha))) + try: + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + return file_contents_ro_filepath(db_path) + except OSError: + raise BadObject(sha) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(sha) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(bin_to_hex(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + + if istream.binsha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies + # END handle custom writer + + try: + try: + if istream.binsha is not None: + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + + hexsha = None + if istream.binsha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha + + if tmp_path: + obj_path = self.db_path(self.object_path(hexsha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + # rename onto existing doesn't work on windows + if os.name == 'nt' and isfile(obj_path): + remove(obj_path) + # END handle win322 + rename(tmp_path, obj_path) + + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + # END handle dry_run + + istream.binsha = hex_to_bin(hexsha) + return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) + diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 5d76c83cc..b9b2b8995 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -5,109 +5,109 @@ """Contains the MemoryDatabase implementation""" from loose import LooseObjectDB from base import ( - ObjectDBR, - ObjectDBW - ) + ObjectDBR, + ObjectDBW + ) from gitdb.base import ( - OStream, - IStream, - ) + OStream, + IStream, + ) from gitdb.exc import ( - BadObject, - UnsupportedOperation - ) + BadObject, + UnsupportedOperation + ) from gitdb.stream import ( - ZippedStoreShaWriter, - DecompressMemMapReader, - ) + ZippedStoreShaWriter, + DecompressMemMapReader, + ) from cStringIO import StringIO __all__ = ("MemoryDB", ) class MemoryDB(ObjectDBR, ObjectDBW): - """A memory database stores everything to memory, providing fast IO and object - retrieval. It should be used to buffer results and obtain SHAs before writing - it to the actual physical storage, as it allows to query whether object already - exists in the target storage before introducing actual IO - - **Note:** memory is currently not threadsafe, hence the async methods cannot be used - for storing""" - - def __init__(self): - super(MemoryDB, self).__init__() - self._db = LooseObjectDB("path/doesnt/matter") - - # maps 20 byte shas to their OStream objects - self._cache = dict() - - def set_ostream(self, stream): - raise UnsupportedOperation("MemoryDB's always stream into memory") - - def store(self, istream): - zstream = ZippedStoreShaWriter() - self._db.set_ostream(zstream) - - istream = self._db.store(istream) - zstream.close() # close to flush - zstream.seek(0) - - # don't provide a size, the stream is written in object format, hence the - # header needs decompression - decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) - self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) - - return istream - - def store_async(self, reader): - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - - def has_object(self, sha): - return sha in self._cache + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO + + **Note:** memory is currently not threadsafe, hence the async methods cannot be used + for storing""" + + def __init__(self): + super(MemoryDB, self).__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) + + return istream + + def store_async(self, reader): + raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") + + def has_object(self, sha): + return sha in self._cache - def info(self, sha): - # we always return streams, which are infos as well - return self.stream(sha) - - def stream(self, sha): - try: - ostream = self._cache[sha] - # rewind stream for the next one to read - ostream.stream.seek(0) - return ostream - except KeyError: - raise BadObject(sha) - # END exception handling - - def size(self): - return len(self._cache) - - def sha_iter(self): - return self._cache.iterkeys() - - - #{ Interface - def stream_copy(self, sha_iter, odb): - """Copy the streams as identified by sha's yielded by sha_iter into the given odb - The streams will be copied directly - **Note:** the object will only be written if it did not exist in the target db - :return: amount of streams actually copied into odb. If smaller than the amount - of input shas, one or more objects did already exist in odb""" - count = 0 - for sha in sha_iter: - if odb.has_object(sha): - continue - # END check object existance - - ostream = self.stream(sha) - # compressed data including header - sio = StringIO(ostream.stream.data()) - istream = IStream(ostream.type, ostream.size, sio, sha) - - odb.store(istream) - count += 1 - # END for each sha - return count - #} END interface + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError: + raise BadObject(sha) + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.iterkeys() + + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + **Note:** the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existance + + ostream = self.stream(sha) + # compressed data including header + sio = StringIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 4c9d0b919..928731937 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -4,18 +4,18 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" from base import ( - FileDBBase, - ObjectDBR, - CachingDB - ) + FileDBBase, + ObjectDBR, + CachingDB + ) from gitdb.util import LazyMixin from gitdb.exc import ( - BadObject, - UnsupportedOperation, - AmbiguousObjectName - ) + BadObject, + UnsupportedOperation, + AmbiguousObjectName + ) from gitdb.pack import PackEntity @@ -28,182 +28,182 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): - """A database operating on a set of object packs""" - - # sort the priority list every N queries - # Higher values are better, performance tests don't show this has - # any effect, but it should have one - _sort_interval = 500 - - def __init__(self, root_path): - super(PackedDB, self).__init__(root_path) - # list of lists with three items: - # * hits - number of times the pack was hit with a request - # * entity - Pack entity instance - # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query - # self._entities = list() # lazy loaded list - self._hit_count = 0 # amount of hits - self._st_mtime = 0 # last modification data of our root path - - def _set_cache_(self, attr): - if attr == '_entities': - self._entities = list() - self.update_cache(force=True) - # END handle entities initialization - - def _sort_entities(self): - self._entities.sort(key=lambda l: l[0], reverse=True) - - def _pack_info(self, sha): - """:return: tuple(entity, index) for an item at the given sha - :param sha: 20 or 40 byte sha - :raise BadObject: - **Note:** This method is not thread-safe, but may be hit in multi-threaded - operation. The worst thing that can happen though is a counter that - was not incremented, or the list being in wrong order. So we safe - the time for locking here, lets see how that goes""" - # presort ? - if self._hit_count % self._sort_interval == 0: - self._sort_entities() - # END update sorting - - for item in self._entities: - index = item[2](sha) - if index is not None: - item[0] += 1 # one hit for you - self._hit_count += 1 # general hit count - return (item[1], index) - # END index found in pack - # END for each item - - # no hit, see whether we have to update packs - # NOTE: considering packs don't change very often, we safe this call - # and leave it to the super-caller to trigger that - raise BadObject(sha) - - #{ Object DB Read - - def has_object(self, sha): - try: - self._pack_info(sha) - return True - except BadObject: - return False - # END exception handling - - def info(self, sha): - entity, index = self._pack_info(sha) - return entity.info_at_index(index) - - def stream(self, sha): - entity, index = self._pack_info(sha) - return entity.stream_at_index(index) - - def sha_iter(self): - sha_list = list() - for entity in self.entities(): - index = entity.index() - sha_by_index = index.sha - for index in xrange(index.size()): - yield sha_by_index(index) - # END for each index - # END for each entity - - def size(self): - sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes, 0) - - #} END object db read - - #{ object db write - - def store(self, istream): - """Storing individual objects is not feasible as a pack is designed to - hold multiple objects. Writing or rewriting packs for single objects is - inefficient""" - raise UnsupportedOperation() - - def store_async(self, reader): - # TODO: add ObjectDBRW before implementing this - raise NotImplementedError() - - #} END object db write - - - #{ Interface - - def update_cache(self, force=False): - """ - Update our cache with the acutally existing packs on disk. Add new ones, - and remove deleted ones. We keep the unchanged ones - - :param force: If True, the cache will be updated even though the directory - does not appear to have changed according to its modification timestamp. - :return: True if the packs have been updated so there is new information, - False if there was no change to the pack database""" - stat = os.stat(self.root_path()) - if not force and stat.st_mtime <= self._st_mtime: - return False - # END abort early on no change - self._st_mtime = stat.st_mtime - - # packs are supposed to be prefixed with pack- by git-convention - # get all pack files, figure out what changed - pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) - our_pack_files = set(item[1].pack().path() for item in self._entities) - - # new packs - for pack_file in (pack_files - our_pack_files): - # init the hit-counter/priority with the size, a good measure for hit- - # probability. Its implemented so that only 12 bytes will be read - entity = PackEntity(pack_file) - self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) - # END for each new packfile - - # removed packs - for pack_file in (our_pack_files - pack_files): - del_index = -1 - for i, item in enumerate(self._entities): - if item[1].pack().path() == pack_file: - del_index = i - break - # END found index - # END for each entity - assert del_index != -1 - del(self._entities[del_index]) - # END for each removed pack - - # reinitialize prioritiess - self._sort_entities() - return True - - def entities(self): - """:return: list of pack entities operated upon by this database""" - return [ item[1] for item in self._entities ] - - def partial_to_complete_sha(self, partial_binsha, canonical_length): - """:return: 20 byte sha as inferred by the given partial binary sha - :param partial_binsha: binary sha with less than 20 bytes - :param canonical_length: length of the corresponding canonical representation. - It is required as binary sha's cannot display whether the original hex sha - had an odd or even number of characters - :raise AmbiguousObjectName: - :raise BadObject: """ - candidate = None - for item in self._entities: - item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) - if item_index is not None: - sha = item[1].index().sha(item_index) - if candidate and candidate != sha: - raise AmbiguousObjectName(partial_binsha) - candidate = sha - # END handle full sha could be found - # END for each entity - - if candidate: - return candidate - - # still not found ? - raise BadObject(partial_binsha) - - #} END interface + """A database operating on a set of object packs""" + + # sort the priority list every N queries + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 + + def __init__(self, root_path): + super(PackedDB, self).__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + if attr == '_entities': + self._entities = list() + self.update_cache(force=True) + # END handle entities initialization + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + **Note:** This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) + + #{ Object DB Read + + def has_object(self, sha): + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling + + def info(self, sha): + entity, index = self._pack_info(sha) + return entity.info_at_index(index) + + def stream(self, sha): + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) + + def sha_iter(self): + sha_list = list() + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes, 0) + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + def store_async(self, reader): + # TODO: add ObjectDBRW before implementing this + raise NotImplementedError() + + #} END object db write + + + #{ Interface + + def update_cache(self, force=False): + """ + Update our cache with the acutally existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = set(item[1].pack().path() for item in self._entities) + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [ item[1] for item in self._entities ] + + def partial_to_complete_sha(self, partial_binsha, canonical_length): + """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) + + #} END interface diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 898984323..60004a77a 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -3,77 +3,77 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - CompoundDB, - ) + CompoundDB, + ) import os __all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): - """A database consisting of database referred to in a file""" - - # Configuration - # Specifies the object database to use for the paths found in the alternates - # file. If None, it defaults to the GitDB - ObjectDBCls = None - - def __init__(self, ref_file): - super(ReferenceDB, self).__init__() - self._ref_file = ref_file - - def _set_cache_(self, attr): - if attr == '_dbs': - self._dbs = list() - self._update_dbs_from_ref_file() - else: - super(ReferenceDB, self)._set_cache_(attr) - # END handle attrs - - def _update_dbs_from_ref_file(self): - dbcls = self.ObjectDBCls - if dbcls is None: - # late import - from git import GitDB - dbcls = GitDB - # END get db type - - # try to get as many as possible, don't fail if some are unavailable - ref_paths = list() - try: - ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] - except (OSError, IOError): - pass - # END handle alternates - - ref_paths_set = set(ref_paths) - cur_ref_paths_set = set(db.root_path() for db in self._dbs) - - # remove existing - for path in (cur_ref_paths_set - ref_paths_set): - for i, db in enumerate(self._dbs[:]): - if db.root_path() == path: - del(self._dbs[i]) - continue - # END del matching db - # END for each path to remove - - # add new - # sort them to maintain order - added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) - for path in added_paths: - try: - db = dbcls(path) - # force an update to verify path - if isinstance(db, CompoundDB): - db.databases() - # END verification - self._dbs.append(db) - except Exception, e: - # ignore invalid paths or issues - pass - # END for each path to add - - def update_cache(self, force=False): - # re-read alternates and update databases - self._update_dbs_from_ref_file() - return super(ReferenceDB, self).update_cache(force) + """A database consisting of database referred to in a file""" + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super(ReferenceDB, self).__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + else: + super(ReferenceDB, self)._set_cache_(attr) + # END handle attrs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + except (OSError, IOError): + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = set(db.root_path() for db in self._dbs) + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception, e: + # ignore invalid paths or issues + pass + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super(ReferenceDB, self).update_cache(force) diff --git a/gitdb/exc.py b/gitdb/exc.py index e087047b4..7180fb586 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -6,27 +6,27 @@ from util import to_hex_sha class ODBError(Exception): - """All errors thrown by the object database""" - + """All errors thrown by the object database""" + class InvalidDBRoot(ODBError): - """Thrown if an object database cannot be initialized at the given path""" - + """Thrown if an object database cannot be initialized at the given path""" + class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the - failed sha""" - - def __str__(self): - return "BadObject: %s" % to_hex_sha(self.args[0]) - + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) + class ParseError(ODBError): - """Thrown if the parsing of a file failed due to an invalid format""" + """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): - """Thrown if a possibly shortened name does not uniquely represent a single object - in the database""" + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" class BadObjectType(ODBError): - """The object had an unsupported type""" + """The object had an unsupported type""" class UnsupportedOperation(ODBError): - """Thrown if the given operation cannot be supported by the object database""" + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/ext/async b/gitdb/ext/async index 039c1d5c2..571412931 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 039c1d5c26bc2ceaa9e55082efae2068d9873e45 +Subproject commit 571412931829200aff06a44b9c5524e122e524e9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 360a8956f..1b3ab5598 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 360a8956fe73a0a96315e946f52737569d990369 +Subproject commit 1b3ab5598e93369282502d049d64cb2ca12839cb diff --git a/gitdb/fun.py b/gitdb/fun.py index 66130ebee..c1e73e895 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -7,8 +7,8 @@ it into c later, if required""" from exc import ( - BadObjectType - ) + BadObjectType + ) from util import zlib decompressobj = zlib.decompressobj @@ -23,655 +23,655 @@ REF_DELTA = 7 delta_types = (OFS_DELTA, REF_DELTA) -type_id_to_type_map = { - 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", - 5 : "", # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA - } +type_id_to_type_map = { + 0 : "", # EXT 1 + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA + } type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA - ) + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA + ) # used when dealing with larger streams chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures def _set_delta_rbound(d, size): - """Truncate the given delta to the given size - :param size: size relative to our target offset, may not be 0, must be smaller or equal - to our size - :return: d""" - d.ts = size - - # NOTE: data is truncated automatically when applying the delta - # MUST NOT DO THIS HERE - return d - + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size + :return: d""" + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE + return d + def _move_delta_lbound(d, bytes): - """Move the delta by the given amount of bytes, reducing its size so that its - right bound stays static - :param bytes: amount of bytes to move, must be smaller than delta size - :return: d""" - if bytes == 0: - return - - d.to += bytes - d.so += bytes - d.ts -= bytes - if d.data is not None: - d.data = d.data[bytes:] - # END handle data - - return d - + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return + + d.to += bytes + d.so += bytes + d.ts -= bytes + if d.data is not None: + d.data = d.data[bytes:] + # END handle data + + return d + def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data) - + return DeltaChunk(src.to, src.ts, src.so, src.data) + def delta_chunk_apply(dc, bbuf, write): - """Apply own data to the target buffer - :param bbuf: buffer providing source bytes for copy operations - :param write: write method to call with data to write""" - if dc.data is None: - # COPY DATA FROM SOURCE - write(buffer(bbuf, dc.so, dc.ts)) - else: - # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? - # Considering data can be larger than 127 bytes now, it should be worth it - if dc.ts < len(dc.data): - write(dc.data[:dc.ts]) - else: - write(dc.data) - # END handle truncation - # END handle chunk mode + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(buffer(bbuf, dc.so, dc.ts)) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode class DeltaChunk(object): - """Represents a piece of a delta, it can either add new data, or copy existing - one from a source buffer""" - __slots__ = ( - 'to', # start offset in the target buffer in bytes - 'ts', # size of this chunk in the target buffer in bytes - 'so', # start offset in the source buffer in bytes or None - 'data', # chunk of bytes to be added to the target buffer, - # DeltaChunkList to use as base, or None - ) - - def __init__(self, to, ts, so, data): - self.to = to - self.ts = ts - self.so = so - self.data = data + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data', # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data - def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") - - #{ Interface - - def rbound(self): - return self.to + self.ts - - def has_data(self): - """:return: True if the instance has data to add to the target stream""" - return self.data is not None - - #} END interface + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + + #{ Interface + + def rbound(self): + return self.to + self.ts + + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is not None + + #} END interface def _closest_index(dcl, absofs): - """:return: index at which the given absofs should be inserted. The index points - to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs. - **Note:** global method for performance only, it belongs to DeltaChunkList""" - lo = 0 - hi = len(dcl) - while lo < hi: - mid = (lo + hi) / 2 - dc = dcl[mid] - if dc.to > absofs: - hi = mid - elif dc.rbound() > absofs or dc.to == absofs: - return mid - else: - lo = mid + 1 - # END handle bound - # END for each delta absofs - return len(dcl)-1 - + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs. + **Note:** global method for performance only, it belongs to DeltaChunkList""" + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound + # END for each delta absofs + return len(dcl)-1 + def delta_list_apply(dcl, bbuf, write): - """Apply the chain's changes and write the final result using the passed - write function. - :param bbuf: base buffer containing the base of all deltas contained in this - list. It will only be used if the chunk in question does not have a base - chain. - :param write: function taking a string of bytes to write to the output""" - for dc in dcl: - delta_chunk_apply(dc, bbuf, write) - # END for each dc + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc def delta_list_slice(dcl, absofs, size, ndcl): - """:return: Subsection of this list at the given absolute offset, with the given - size in bytes. - :return: None""" - cdi = _closest_index(dcl, absofs) # delta start index - cd = dcl[cdi] - slen = len(dcl) - lappend = ndcl.append - - if cd.to != absofs: - tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) - _move_delta_lbound(tcd, absofs - cd.to) - tcd.ts = min(tcd.ts, size) - lappend(tcd) - size -= tcd.ts - cdi += 1 - # END lbound overlap handling - - while cdi < slen and size: - # are we larger than the current block - cd = dcl[cdi] - if cd.ts <= size: - lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) - size -= cd.ts - else: - tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) - tcd.ts = size - lappend(tcd) - size -= tcd.ts - break - # END hadle size - cdi += 1 - # END for each chunk - - + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: None""" + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + lappend = ndcl.append + + if cd.to != absofs: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + _move_delta_lbound(tcd, absofs - cd.to) + tcd.ts = min(tcd.ts, size) + lappend(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) + size -= cd.ts + else: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size + lappend(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + class DeltaChunkList(list): - """List with special functionality to deal with DeltaChunks. - There are two types of lists we represent. The one was created bottom-up, working - towards the latest delta, the other kind was created top-down, working from the - latest delta down to the earliest ancestor. This attribute is queryable - after all processing with is_reversed.""" - - __slots__ = tuple() - - def rbound(self): - """:return: rightmost extend in bytes, absolute""" - if len(self) == 0: - return 0 - return self[-1].rbound() - - def lbound(self): - """:return: leftmost byte at which this chunklist starts""" - if len(self) == 0: - return 0 - return self[0].to - - def size(self): - """:return: size of bytes as measured by our delta chunks""" - return self.rbound() - self.lbound() - - def apply(self, bbuf, write): - """Only used by public clients, internally we only use the global routines - for performance""" - return delta_list_apply(self, bbuf, write) - - def compress(self): - """Alter the list to reduce the amount of nodes. Currently we concatenate - add-chunks - :return: self""" - slen = len(self) - if slen < 2: - return self - i = 0 - slen_orig = slen - - first_data_index = None - while i < slen: - dc = self[i] - i += 1 - if dc.data is None: - if first_data_index is not None and i-2-first_data_index > 1: - #if first_data_index is not None: - nd = StringIO() # new data - so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i-1): - xdc = self[x] - nd.write(xdc.data[:xdc.ts]) - # END collect data - - del(self[first_data_index:i-1]) - buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) - - slen = len(self) - i = first_data_index + 1 - - # END concatenate data - first_data_index = None - continue - # END skip non-data chunks - - if first_data_index is None: - first_data_index = i-1 - # END iterate list - - #if slen_orig != len(self): - # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) - return self - - def check_integrity(self, target_size=-1): - """Verify the list has non-overlapping chunks only, and the total size matches - target_size - :param target_size: if not -1, the total size of the chain must be target_size - :raise AssertionError: if the size doen't match""" - if target_size > -1: - assert self[-1].rbound() == target_size - assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size - # END target size verification - - if len(self) < 2: - return - - # check data - for dc in self: - assert dc.ts > 0 - if dc.has_data(): - assert len(dc.data) >= dc.ts - # END for each dc - - left = islice(self, 0, len(self)-1) - right = iter(self) - right.next() - # this is very pythonic - we might have just use index based access here, - # but this could actually be faster - for lft,rgt in izip(left, right): - assert lft.rbound() == rgt.to - assert lft.to + lft.ts == rgt.to - # END for each pair - + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def apply(self, bbuf, write): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write) + + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + slen_orig = slen + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if dc.data is None: + if first_data_index is not None and i-2-first_data_index > 1: + #if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in xrange(first_data_index, i-1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i-1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i-1 + # END iterate list + + #if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doen't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + assert dc.ts > 0 + if dc.has_data(): + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self)-1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft,rgt in izip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + class TopdownDeltaChunkList(DeltaChunkList): - """Represents a list which is generated by feeding its ancestor streams one by - one""" - __slots__ = tuple() - - def connect_with_next_base(self, bdcl): - """Connect this chain with the next level of our base delta chunklist. - The goal in this game is to mark as many of our chunks rigid, hence they - cannot be changed by any of the upcoming bases anymore. Once all our - chunks are marked like that, we can stop all processing - :param bdcl: data chunk list being one of our bases. They must be fed in - consequtively and in order, towards the earliest ancestor delta - :return: True if processing was done. Use it to abort processing of - remaining streams if False is returned""" - nfc = 0 # number of frozen chunks - dci = 0 # delta chunk index - slen = len(self) # len of self - ccl = list() # temporary list - while dci < slen: - dc = self[dci] - dci += 1 - - # all add-chunks which are already topmost don't need additional processing - if dc.data is not None: - nfc += 1 - continue - # END skip add chunks - - # copy chunks - # integrate the portion of the base list into ourselves. Lists - # dont support efficient insertion ( just one at a time ), but for now - # we live with it. Internally, its all just a 32/64bit pointer, and - # the portions of moved memory should be smallish. Maybe we just rebuild - # ourselves in order to reduce the amount of insertions ... - del(ccl[:]) - delta_list_slice(bdcl, dc.so, dc.ts, ccl) - - # move the target bounds into place to match with our chunk - ofs = dc.to - dc.so - for cdc in ccl: - cdc.to += ofs - # END update target bounds - - if len(ccl) == 1: - self[dci-1] = ccl[0] - else: - # maybe try to compute the expenses here, and pick the right algorithm - # It would normally be faster than copying everything physically though - # TODO: Use a deque here, and decide by the index whether to extend - # or extend left ! - post_dci = self[dci:] - del(self[dci-1:]) # include deletion of dc - self.extend(ccl) - self.extend(post_dci) - - slen = len(self) - dci += len(ccl)-1 # deleted dc, added rest - - # END handle chunk replacement - # END for each chunk - - if nfc == slen: - return False - # END handle completeness - return True - - + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = tuple() + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consequtively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams if False is returned""" + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + ccl = list() # temporary list + while dci < slen: + dc = self[dci] + dci += 1 + + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + if len(ccl) == 1: + self[dci-1] = ccl[0] + else: + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci-1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl)-1 # deleted dc, added rest + + # END handle chunk replacement + # END for each chunk + + if nfc == slen: + return False + # END handle completeness + return True + + #} END structures #{ Routines def is_loose_object(m): - """ - :return: True the file contained in memory map m appears to be a loose object. - Only the first two bytes are needed""" - b0, b1 = map(ord, m[:2]) - word = (b0 << 8) + b1 - return b0 == 0x78 and (word % 31) == 0 + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 def loose_object_header_info(m): - """ - :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the - object as well as its uncompressed size in bytes. - :param m: memory map from which to read the compressed object data""" - decompress_size = 8192 # is used in cgit as well - hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0")].split(" ") - return type_name, int(size) - + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find("\0")].split(" ") + return type_name, int(size) + def pack_object_header_info(data): - """ - :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) - The type_id should be interpreted according to the ``type_id_to_type_map`` map - The byte-offset specifies the start of the actual zlib compressed datastream - :param m: random-access memory, like a string or memory map""" - c = ord(data[0]) # first byte - i = 1 # next char to read - type_id = (c >> 4) & 7 # numeric type - size = c & 15 # starting size - s = 4 # starting bit-shift size - while c & 0x80: - c = ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop - return (type_id, size, i) + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" + c = ord(data[0]) # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): - """ - :return: string defining the pack header comprised of the object type - and its incompressed size in bytes - - :param obj_type: pack type_id of the object - :param obj_size: uncompressed size in bytes of the following object stream""" - c = 0 # 1 byte - hdr = str() # output string + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = str() # output string - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - #END until size is consumed - hdr += chr(c) - return hdr - + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + return hdr + def msb_size(data, offset=0): - """ - :return: tuple(read_bytes, size) read the msb size from the given random - access data starting at the given byte offset""" - size = 0 - i = 0 - l = len(data) - hit_msb = False - while i < l: - c = ord(data[i+offset]) - size |= (c & 0x7f) << i*7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range - if not hit_msb: - raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size - + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i+offset, size + def loose_object_header(type, size): - """ - :return: string representing the loose object header, which is immediately - followed by the content stream of size 'size'""" - return "%s %i\0" % (type, size) - + """ + :return: string representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return "%s %i\0" % (type, size) + def write_object(type, size, read, write, chunk_size=chunk_size): - """ - Write the object as identified by type, size and source_stream into the - target_stream - - :param type: type string of the object - :param size: amount of bytes to write from source_stream - :param read: read method of a stream providing the content data - :param write: write method of the output stream - :param close_target_stream: if True, the target stream will be closed when - the routine exits, even if an error is thrown - :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" - tbw = 0 # total num bytes written - - # WRITE HEADER: type SP size NULL - tbw += write(loose_object_header(type, size)) - tbw += stream_copy(read, write, size, chunk_size) - - return tbw + """ + Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write(loose_object_header(type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw def stream_copy(read, write, size, chunk_size): - """ - Copy a stream up to size bytes using the provided read and write methods, - in chunks of chunk_size - - **Note:** its much like stream_copy utility, but operates just using methods""" - dbw = 0 # num data bytes written - - # WRITE ALL DATA UP TO SIZE - while True: - cs = min(chunk_size, size-dbw) - # NOTE: not all write methods return the amount of written bytes, like - # mmap.write. Its bad, but we just deal with it ... perhaps its not - # even less efficient - # data_len = write(read(cs)) - # dbw += data_len - data = read(cs) - data_len = len(data) - dbw += data_len - write(data) - if data_len < cs or dbw == size: - break - # END check for stream end - # END duplicate data - return dbw - + """ + Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + + **Note:** its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size-dbw) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) + dbw += data_len + write(data) + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + def connect_deltas(dstreams): - """ - Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks - - :param dstreams: iterable of delta stream objects, the delta to be applied last - comes first, then all its ancestors in order - :return: DeltaChunkList, containing all operations to apply""" - tdcl = None # topmost dcl - - dcl = tdcl = TopdownDeltaChunkList() - for dsi, ds in enumerate(dstreams): - # print "Stream", dsi - db = ds.read() - delta_buf_size = ds.size - - # read header - i, base_size = msb_size(db) - i, target_size = msb_size(db, i) - - # interpret opcodes - tbw = 0 # amount of target bytes written - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > base_size): - break - - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) - tbw += cp_size - elif c: - # NOTE: in C, the data chunks should probably be concatenated here. - # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) - i += c - tbw += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - - dcl.compress() - - # merge the lists ! - if dsi > 0: - if not tdcl.connect_with_next_base(dcl): - break - # END handle merge - - # prepare next base - dcl = DeltaChunkList() - # END for each delta stream - - return tdcl - + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order + :return: DeltaChunkList, containing all operations to apply""" + tdcl = None # topmost dcl + + dcl = tdcl = TopdownDeltaChunkList() + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi + db = ds.read() + delta_buf_size = ds.size + + # read header + i, base_size = msb_size(db) + i, target_size = msb_size(db, i) + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > base_size): + break + + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.compress() + + # merge the lists ! + if dsi > 0: + if not tdcl.connect_with_next_base(dcl): + break + # END handle merge + + # prepare next base + dcl = DeltaChunkList() + # END for each delta stream + + return tdcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): - """ - Apply data from a delta buffer using a source buffer to the target file - - :param src_buf: random access data from which the delta was created - :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes - :param delta_buf: random access delta data - :param write: write method taking a chunk of bytes - - **Note:** transcribed to python from the similar routine in patch-delta.c""" - i = 0 - db = delta_buf - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i+c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - - # yes, lets use the exact same error message that git uses :) - assert i == delta_buf_size, "delta replay has gone wild" - - + """ + Apply data from a delta buffer using a source buffer to the target file + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf: random access delta data + :param write: write method taking a chunk of bytes + + **Note:** transcribed to python from the similar routine in patch-delta.c""" + i = 0 + db = delta_buf + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + + def is_equal_canonical_sha(canonical_length, match, sha1): - """ - :return: True if the given lhs and rhs 20 byte binary shas - The comparison will take the canonical_length of the match sha into account, - hence the comparison will only use the last 4 bytes for uneven canonical representations - :param match: less than 20 byte sha - :param sha1: 20 byte sha""" - binary_length = canonical_length/2 - if match[:binary_length] != sha1[:binary_length]: - return False - - if canonical_length - binary_length and \ - (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: - return False - # END handle uneven canonnical length - return True - + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length/2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + #} END routines try: - # raise ImportError; # DEBUG - from _perf import connect_deltas + # raise ImportError; # DEBUG + from _perf import connect_deltas except ImportError: - pass + pass diff --git a/gitdb/pack.py b/gitdb/pack.py index c6d1cc313..48121f026 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -4,59 +4,59 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) + BadObject, + UnsupportedOperation, + ParseError + ) from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, + ) from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size + ) try: - from _perf import PackIndexFile_sha_to_index + from _perf import PackIndexFile_sha_to_index except ImportError: - pass + pass # END try c module -from base import ( # Amazing ! - OInfo, - OStream, - OPackInfo, - OPackStream, - ODeltaStream, - ODeltaPackInfo, - ODeltaPackStream, - ) +from base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, + ) from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer + ) from struct import ( - pack, - unpack, - ) + pack, + unpack, + ) from binascii import crc32 @@ -70,943 +70,943 @@ - + #{ Utilities def pack_object_at(cursor, offset, as_stream): - """ - :return: Tuple(abs_data_offset, PackInfo|PackStream) - an object of the correct type according to the type_id of the object. - If as_stream is True, the object will contain a stream, allowing the - data to be read decompressed. - :param data: random accessable data containing all required information - :parma offset: offset in to the data at which the object information is located - :param as_stream: if True, a stream object will be returned that can read - the data, otherwise you receive an info object only""" - data = cursor.use_region(offset).buffer() - type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) - total_rela_offset = None # set later, actual offset until data stream begins - delta_info = None - - # OFFSET DELTA - if type_id == OFS_DELTA: - i = data_rela_offset - c = ord(data[i]) - i += 1 - delta_offset = c & 0x7f - while c & 0x80: - c = ord(data[i]) - i += 1 - delta_offset += 1 - delta_offset = (delta_offset << 7) + (c & 0x7f) - # END character loop - delta_info = delta_offset - total_rela_offset = i - # REF DELTA - elif type_id == REF_DELTA: - total_rela_offset = data_rela_offset+20 - delta_info = data[data_rela_offset:total_rela_offset] - # BASE OBJECT - else: - # assume its a base object - total_rela_offset = data_rela_offset - # END handle type id - - abs_data_offset = offset + total_rela_offset - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) - if delta_info is None: - return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) - else: - return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) - else: - if delta_info is None: - return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) - else: - return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) - # END handle info - # END handle stream + """ + :return: Tuple(abs_data_offset, PackInfo|PackStream) + an object of the correct type according to the type_id of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. + :param data: random accessable data containing all required information + :parma offset: offset in to the data at which the object information is located + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only""" + data = cursor.use_region(offset).buffer() + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA + if type_id == OFS_DELTA: + i = data_rela_offset + c = ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: + c = ord(data[i]) + i += 1 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) + # END character loop + delta_info = delta_offset + total_rela_offset = i + # REF DELTA + elif type_id == REF_DELTA: + total_rela_offset = data_rela_offset+20 + delta_info = data[data_rela_offset:total_rela_offset] + # BASE OBJECT + else: + # assume its a base object + total_rela_offset = data_rela_offset + # END handle type id + + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + if delta_info is None: + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) + else: + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) + else: + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream def write_stream_to_pack(read, write, zstream, base_crc=None): - """Copy a stream as read from read function, zip it, and write the result. - Count the number of written bytes and return it - :param base_crc: if not None, the crc will be the base for all compressed data - we consecutively write and generate a crc32 from. If None, no crc will be generated - :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc - was false""" - br = 0 # bytes read - bw = 0 # bytes written - want_crc = base_crc is not None - crc = 0 - if want_crc: - crc = base_crc - #END initialize crc - - while True: - chunk = read(chunk_size) - br += len(chunk) - compressed = zstream.compress(chunk) - bw += len(compressed) - write(compressed) # cannot assume return value - - if want_crc: - crc = crc32(compressed, crc) - #END handle crc - - if len(chunk) != chunk_size: - break - #END copy loop - - compressed = zstream.flush() - bw += len(compressed) - write(compressed) - if want_crc: - crc = crc32(compressed, crc) - #END handle crc - - return (br, bw, crc) + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + want_crc = base_crc is not None + crc = 0 + if want_crc: + crc = base_crc + #END initialize crc + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + if len(chunk) != chunk_size: + break + #END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + return (br, bw, crc) #} END utilities class IndexWriter(object): - """Utility to cache index information, allowing to write all information later - in one go to the given stream - **Note:** currently only writes v2 indices""" - __slots__ = '_objs' - - def __init__(self): - self._objs = list() - - def append(self, binsha, crc, offset): - """Append one piece of object information""" - self._objs.append((binsha, crc, offset)) - - def write(self, pack_sha, write): - """Write the index file using the given write method - :param pack_sha: binary sha over the whole pack that we index - :return: sha1 binary sha over all index file contents""" - # sort for sha1 hash - self._objs.sort(key=lambda o: o[0]) - - sha_writer = FlexibleSha1Writer(write) - sha_write = sha_writer.write - sha_write(PackIndexFile.index_v2_signature) - sha_write(pack(">L", PackIndexFile.index_version_default)) - - # fanout - tmplist = list((0,)*256) # fanout or list with 64 bit offsets - for t in self._objs: - tmplist[ord(t[0][0])] += 1 - #END prepare fanout - for i in xrange(255): - v = tmplist[i] - sha_write(pack('>L', v)) - tmplist[i+1] += v - #END write each fanout entry - sha_write(pack('>L', tmplist[255])) - - # sha1 ordered - # save calls, that is push them into c - sha_write(''.join(t[0] for t in self._objs)) - - # crc32 - for t in self._objs: - sha_write(pack('>L', t[1]&0xffffffff)) - #END for each crc - - tmplist = list() - # offset 32 - for t in self._objs: - ofs = t[2] - if ofs > 0x7fffffff: - tmplist.append(ofs) - ofs = 0x80000000 + len(tmplist)-1 - #END hande 64 bit offsets - sha_write(pack('>L', ofs&0xffffffff)) - #END for each offset - - # offset 64 - for ofs in tmplist: - sha_write(pack(">Q", ofs)) - #END for each offset - - # trailer - assert(len(pack_sha) == 20) - sha_write(pack_sha) - sha = sha_writer.sha(as_hex=False) - write(sha) - return sha - - + """Utility to cache index information, allowing to write all information later + in one go to the given stream + **Note:** currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_sha, write): + """Write the index file using the given write method + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,)*256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[ord(t[0][0])] += 1 + #END prepare fanout + for i in xrange(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i+1] += v + #END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1]&0xffffffff)) + #END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist)-1 + #END hande 64 bit offsets + sha_write(pack('>L', ofs&0xffffffff)) + #END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + #END for each offset + + # trailer + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha + + class PackIndexFile(LazyMixin): - """A pack index provides offsets into the corresponding pack, allowing to find - locations for offsets faster.""" - - # Dont use slots as we dynamically bind functions for each version, need a dict for this - # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', - # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') - # used in v2 indices - _sha_list_offset = 8 + 1024 - index_v2_signature = '\377tOc' - index_version_default = 2 + # used in v2 indices + _sha_list_offset = 8 + 1024 + index_v2_signature = '\377tOc' + index_version_default = 2 - def __init__(self, indexpath): - super(PackIndexFile, self).__init__() - self._indexpath = indexpath - - def _set_cache_(self, attr): - if attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-40:-20] - elif attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-20:] - elif attr == "_cursor": - # Note: We don't lock the file when reading as we cannot be sure - # that we can actually write to the location - it could be a read-only - # alternate for instance - self._cursor = mman.make_cursor(self._indexpath).use_region() - # We will assume that the index will always fully fit into memory ! - if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): - raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) - #END assert window size - else: - # now its time to initialize everything - if we are here, someone wants - # to access the fanout table or related properties - - # CHECK VERSION - mmap = self._cursor.map() - self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 - if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] - assert version_id == self._version, "Unsupported index version: %i" % version_id - # END assert version - - # SETUP FUNCTIONS - # setup our functions according to the actual version - for fname in ('entry', 'offset', 'sha', 'crc'): - setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) - # END for each function to initialize - - - # INITIALIZE DATA - # byte offset is 8 if version is 2, 0 otherwise - self._initialize() - # END handle attributes - + def __init__(self, indexpath): + super(PackIndexFile, self).__init__() + self._indexpath = indexpath + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-40:-20] + elif attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-20:] + elif attr == "_cursor": + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) + #END assert window size + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", mmap, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + - #{ Access V1 - - def _entry_v1(self, i): - """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - - def _offset_v1(self, i): - """see ``_offset_v2``""" - return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - - def _sha_v1(self, i): - """see ``_sha_v2``""" - base = 1024 + (i*24)+4 - return self._cursor.map()[base:base+20] - - def _crc_v1(self, i): - """unsupported""" - return 0 - - #} END access V1 - - #{ Access V2 - def _entry_v2(self, i): - """:return: tuple(offset, binsha, crc)""" - return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - - def _offset_v2(self, i): - """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only - be returned if the pack is larger than 4 GiB, or 2^32""" - offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - - # if the high-bit is set, this indicates that we have to lookup the offset - # in the 64 bit region of the file. The current offset ( lower 31 bits ) - # are the index into it - if offset & 0x80000000: - offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] - # END handle 64 bit offset - - return offset - - def _sha_v2(self, i): - """:return: sha at the given index of this file index instance""" - base = self._sha_list_offset + i * 20 - return self._cursor.map()[base:base+20] - - def _crc_v2(self, i): - """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - - #} END access V2 - - #{ Initialization - - def _initialize(self): - """initialize base data""" - self._fanout_table = self._read_fanout((self._version == 2) * 8) - - if self._version == 2: - self._crc_list_offset = self._sha_list_offset + self.size() * 20 - self._pack_offset = self._crc_list_offset + self.size() * 4 - self._pack_64_offset = self._pack_offset + self.size() * 4 - # END setup base - - def _read_fanout(self, byte_offset): - """Generate a fanout table from our data""" - d = self._cursor.map() - out = list() - append = out.append - for i in range(256): - append(unpack_from('>L', d, byte_offset + i*4)[0]) - # END for each entry - return out - - #} END initialization - - #{ Properties - def version(self): - return self._version - - def size(self): - """:return: amount of objects referred to by this index""" - return self._fanout_table[255] - - def path(self): - """:return: path to the packindexfile""" - return self._indexpath - - def packfile_checksum(self): - """:return: 20 byte sha representing the sha1 hash of the pack file""" - return self._cursor.map()[-40:-20] - - def indexfile_checksum(self): - """:return: 20 byte sha representing the sha1 hash of this index file""" - return self._cursor.map()[-20:] - - def offsets(self): - """:return: sequence of all offsets in the order in which they were written - - **Note:** return value can be random accessed, but may be immmutable""" - if self._version == 2: - # read stream to array, convert to tuple - a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - - # networkbyteorder to something array likes more - if sys.byteorder == 'little': - a.byteswap() - return a - else: - return tuple(self.offset(index) for index in xrange(self.size())) - # END handle version - - def sha_to_index(self, sha): - """ - :return: index usable with the ``offset`` or ``entry`` method, or None - if the sha was not found in this pack index - :param sha: 20 byte sha to lookup""" - first_byte = ord(sha[0]) - get_sha = self.sha - lo = 0 # lower index, the left bound of the bisection - if first_byte != 0: - lo = self._fanout_table[first_byte-1] - hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - - # bisect until we have the sha - while lo < hi: - mid = (lo + hi) / 2 - c = cmp(sha, get_sha(mid)) - if c < 0: - hi = mid - elif not c: - return mid - else: - lo = mid + 1 - # END handle midpoint - # END bisect - return None - - def partial_sha_to_index(self, partial_bin_sha, canonical_length): - """ - :return: index as in `sha_to_index` or None if the sha was not found in this - index file - :param partial_bin_sha: an at least two bytes of a partial binary sha - :param canonical_length: lenght of the original hexadecimal representation of the - given partial binary sha - :raise AmbiguousObjectName:""" - if len(partial_bin_sha) < 2: - raise ValueError("Require at least 2 bytes of partial sha") - - first_byte = ord(partial_bin_sha[0]) - get_sha = self.sha - lo = 0 # lower index, the left bound of the bisection - if first_byte != 0: - lo = self._fanout_table[first_byte-1] - hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - - # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest - while lo < hi: - mid = (lo + hi) / 2 - c = cmp(filled_sha, get_sha(mid)) - if c < 0: - hi = mid - elif not c: - # perfect match - lo = mid - break - else: - lo = mid + 1 - # END handle midpoint - # END bisect - - if lo < self.size(): - cur_sha = get_sha(lo) - if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): - next_sha = None - if lo+1 < self.size(): - next_sha = get_sha(lo+1) - if next_sha and next_sha == cur_sha: - raise AmbiguousObjectName(partial_bin_sha) - return lo - # END if we have a match - # END if we found something - return None - - if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr - # accesses - def sha_to_index(self, sha): - return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - - #} END properties - - + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + (i*24)+4 + return self._cursor.map()[base:base+20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._cursor.map()[base:base+20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._cursor.map() + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i*4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + def version(self): + return self._version + + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._cursor.map()[-40:-20] + + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._cursor.map()[-20:] + + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + + **Note:** return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + + # networkbyteorder to something array likes more + if sys.byteorder == 'little': + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in xrange(self.size())) + # END handle version + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = ord(sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + return mid + else: + lo = mid + 1 + # END handle midpoint + # END bisect + return None + + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha + :param canonical_length: lenght of the original hexadecimal representation of the + given partial binary sha + :raise AmbiguousObjectName:""" + if len(partial_bin_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + first_byte = ord(partial_bin_sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # fill the partial to full 20 bytes + filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) + + # find lowest + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(filled_sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + + if lo < self.size(): + cur_sha = get_sha(lo) + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): + next_sha = None + if lo+1 < self.size(): + next_sha = get_sha(lo+1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_bin_sha) + return lo + # END if we have a match + # END if we found something + return None + + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + + #} END properties + + class PackFile(LazyMixin): - """A pack is a file written according to the Version 2 for git packs - - As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be - fine though. - - **Note:** at some point, this might be implemented using streams as well, or - streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that - case""" - - __slots__ = ('_packpath', '_cursor', '_size', '_version') - pack_signature = 0x5041434b # 'PACK' - pack_version_default = 2 - - # offset into our data at which the first object starts - first_object_offset = 3*4 # header bytes - footer_size = 20 # final sha - - def __init__(self, packpath): - self._packpath = packpath - - def _set_cache_(self, attr): - # we fill the whole cache, whichever attribute gets queried first - self._cursor = mman.make_cursor(self._packpath).use_region() - - # read the header information - type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - if type_id != self.pack_signature: - raise ParseError("Invalid pack signature: %i" % type_id) - - def _iter_objects(self, start_offset, as_stream=True): - """Handle the actual iteration of objects within this pack""" - c = self._cursor - content_size = c.file_size() - self.footer_size - cur_offset = start_offset or self.first_object_offset - - null = NullStream() - while cur_offset < content_size: - data_offset, ostream = pack_object_at(c, cur_offset, True) - # scrub the stream to the end - this decompresses the object, but yields - # the amount of compressed bytes we need to get to the next offset - - stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - - # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the - # info object - if as_stream: - ostream.stream.seek(0) - yield ostream - # END until we have read everything - - #{ Pack Information - - def size(self): - """:return: The amount of objects stored in this pack""" - return self._size - - def version(self): - """:return: the version of this pack""" - return self._version - - def data(self): - """ - :return: read-only data of this pack. It provides random access and usually - is a memory map. - :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" - # can use map as we are starting at offset 0. Otherwise we would have to use buffer() - return self._cursor.use_region().map() - - def checksum(self): - """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - - def path(self): - """:return: path to the packfile""" - return self._packpath - #} END pack information - - #{ Pack Specific - - def collect_streams(self, offset): - """ - :return: list of pack streams which are required to build the object - at the given offset. The first entry of the list is the object at offset, - the last one is either a full object, or a REF_Delta stream. The latter - type needs its reference object to be locked up in an ODB to form a valid - delta chain. - If the object at offset is no delta, the size of the list is 1. - :param offset: specifies the first byte of the object within this pack""" - out = list() - c = self._cursor - while True: - ostream = pack_object_at(c, offset, True)[1] - out.append(ostream) - if ostream.type_id == OFS_DELTA: - offset = ostream.pack_offset - ostream.delta_info - else: - # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter - # case someone else has to find it - break - # END handle type - # END while chaining streams - return out + """A pack is a file written according to the Version 2 for git packs + + As we currently use memory maps, it could be assumed that the maximum size of + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_cursor', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 + + # offset into our data at which the first object starts + first_object_offset = 3*4 # header bytes + footer_size = 20 # final sha + + def __init__(self, packpath): + self._packpath = packpath + + def _set_cache_(self, attr): + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + + def _iter_objects(self, start_offset, as_stream=True): + """Handle the actual iteration of objects within this pack""" + c = self._cursor + content_size = c.file_size() - self.footer_size + cur_offset = start_offset or self.first_object_offset + + null = NullStream() + while cur_offset < content_size: + data_offset, ostream = pack_object_at(c, cur_offset, True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream + # END until we have read everything + + #{ Pack Information + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def data(self): + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] + + def path(self): + """:return: path to the packfile""" + return self._packpath + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + If the object at offset is no delta, the size of the list is 1. + :param offset: specifies the first byte of the object within this pack""" + out = list() + c = self._cursor + while True: + ostream = pack_object_at(c, offset, True)[1] + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out - #} END pack specific - - #{ Read-Database like Interface - - def info(self, offset): - """Retrieve information about the object at the given file-absolute offset - - :param offset: byte offset - :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - - def stream(self, offset): - """Retrieve an object at the given file-relative offset as stream along with its information - - :param offset: byte offset - :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - - def stream_iter(self, start_offset=0): - """ - :return: iterator yielding OPackStream compatible instances, allowing - to access the data in the pack directly. - :param start_offset: offset to the first object to iterate. If 0, iteration - starts at the very first object in the pack. - - **Note:** Iterating a pack directly is costly as the datastream has to be decompressed - to determine the bounds between the objects""" - return self._iter_objects(start_offset, as_stream=True) - - #} END Read-Database like Interface - - + #} END pack specific + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] + + def stream_iter(self, start_offset=0): + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + + #} END Read-Database like Interface + + class PackEntity(LazyMixin): - """Combines the PackIndexFile and the PackFile into one, allowing the - actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file - '_pack', # our pack file - '_offset_map' # on demand dict mapping one offset to the next consecutive one - ) - - IndexFileCls = PackIndexFile - PackFileCls = PackFile - - def __init__(self, pack_or_index_path): - """Initialize ourselves with the path to the respective pack or index file""" - basename, ext = os.path.splitext(pack_or_index_path) - self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance - self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - - def _set_cache_(self, attr): - # currently this can only be _offset_map - # TODO: make this a simple sorted offset array which can be bisected - # to find the respective entry, from which we can take a +1 easily - # This might be slower, but should also be much lighter in memory ! - offsets_sorted = sorted(self._index.offsets()) - last_offset = len(self._pack.data()) - self._pack.footer_size - assert offsets_sorted, "Cannot handle empty indices" - - offset_map = None - if len(offsets_sorted) == 1: - offset_map = { offsets_sorted[0] : last_offset } - else: - iter_offsets = iter(offsets_sorted) - iter_offsets_plus_one = iter(offsets_sorted) - iter_offsets_plus_one.next() - consecutive = izip(iter_offsets, iter_offsets_plus_one) - - offset_map = dict(consecutive) - - # the last offset is not yet set - offset_map[offsets_sorted[-1]] = last_offset - # END handle offset amount - self._offset_map = offset_map - - def _sha_to_index(self, sha): - """:return: index for the given sha, or raise""" - index = self._index.sha_to_index(sha) - if index is None: - raise BadObject(sha) - return index - - def _iter_objects(self, as_stream): - """Iterate over all objects in our index and yield their OInfo or OStream instences""" - _sha = self._index.sha - _object = self._object - for index in xrange(self._index.size()): - yield _object(_sha(index), as_stream, index) - # END for each index - - def _object(self, sha, as_stream, index=-1): - """:return: OInfo or OStream object providing information about the given sha - :param index: if not -1, its assumed to be the sha's index in the IndexFile""" - # its a little bit redundant here, but it needs to be efficient - if index < 0: - index = self._sha_to_index(sha) - if sha is None: - sha = self._index.sha(index) - # END assure sha is present ( in output ) - offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) - if as_stream: - if type_id not in delta_types: - packstream = self._pack.stream(offset) - return OStream(sha, packstream.type, packstream.size, packstream.stream) - # END handle non-deltas - - # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, - # we extract it from the delta stream ourselves - streams = self.collect_streams_at_offset(offset) - dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) - else: - if type_id not in delta_types: - return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) - # END handle non-deltas - - # deltas are a little tougher - unpack the first bytes to obtain - # the actual target size, as opposed to the size of the delta data - streams = self.collect_streams_at_offset(offset) - buf = streams[0].read(512) - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - - # collect the streams to obtain the actual object type - if streams[-1].type_id in delta_types: - raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) - # END handle stream - - #{ Read-Database like Interface - - def info(self, sha): - """Retrieve information about the object identified by the given sha - - :param sha: 20 byte sha1 - :raise BadObject: - :return: OInfo instance, with 20 byte sha""" - return self._object(sha, False) - - def stream(self, sha): - """Retrieve an object stream along with its information as identified by the given sha - - :param sha: 20 byte sha1 - :raise BadObject: - :return: OStream instance, with 20 byte sha""" - return self._object(sha, True) + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ( '_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def _set_cache_(self, attr): + # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = { offsets_sorted[0] : last_offset } + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + iter_offsets_plus_one.next() + consecutive = izip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map + + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + + def _iter_objects(self, as_stream): + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + _sha = self._index.sha + _object = self._object + for index in xrange(self._index.size()): + yield _object(_sha(index), as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" + # its a little bit redundant here, but it needs to be efficient + if index < 0: + index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) + offset = self._index.offset(index) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + dstream = DeltaApplyReader.new(streams) + + return ODeltaStream(sha, dstream.type, None, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + return OInfo(sha, streams[-1].type, target_size) + # END handle stream + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OInfo instance, with 20 byte sha""" + return self._object(sha, False) + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OStream instance, with 20 byte sha""" + return self._object(sha, True) - def info_at_index(self, index): - """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" - return self._object(None, False, index) - - def stream_at_index(self, index): - """As ``stream``, but uses a PackIndexFile compatible index to refer to the - object""" - return self._object(None, True, index) - - #} END Read-Database like Interface - - #{ Interface + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) + + #} END Read-Database like Interface + + #{ Interface - def pack(self): - """:return: the underlying pack file instance""" - return self._pack - - def index(self): - """:return: the underlying pack index file instance""" - return self._index - - def is_valid_stream(self, sha, use_crc=False): - """ - Verify that the stream at the given sha is valid. - - :param use_crc: if True, the index' crc is run over the compressed stream of - the object, which is much faster than checking the sha1. It is also - more prone to unnoticed corruption or manipulation. - :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is - a delta, this only verifies that the delta's data is valid, not the - data of the actual undeltified object, as it depends on more than - just this stream. - If False, the object will be decompressed and the sha generated. It must - match the given sha - - :return: True if the stream is valid - :raise UnsupportedOperation: If the index is version 1 only - :raise BadObject: sha was not found""" - if use_crc: - if self._index.version() < 2: - raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") - # END handle index version - - index = self._sha_to_index(sha) - offset = self._index.offset(index) - next_offset = self._offset_map[offset] - crc_value = self._index.crc(index) - - # create the current crc value, on the compressed object data - # Read it in chunks, without copying the data - crc_update = zlib.crc32 - pack_data = self._pack.data() - cur_pos = offset - this_crc_value = 0 - while cur_pos < next_offset: - rbound = min(cur_pos + chunk_size, next_offset) - size = rbound - cur_pos - this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) - cur_pos += size - # END window size loop - - # crc returns signed 32 bit numbers, the AND op forces it into unsigned - # mode ... wow, sneaky, from dulwich. - return (this_crc_value & 0xffffffff) == crc_value - else: - shawriter = Sha1Writer() - stream = self._object(sha, as_stream=True) - # write a loose object, which is the basis for the sha - write_object(stream.type, stream.size, stream.read, shawriter.write) - - assert shawriter.sha(as_hex=False) == sha - return shawriter.sha(as_hex=False) == sha - # END handle crc/sha verification - return True + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """ + Verify that the stream at the given sha is valid. + + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + + index = self._sha_to_index(sha) + offset = self._index.offset(index) + next_offset = self._offset_map[offset] + crc_value = self._index.crc(index) + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() + cur_pos = offset + this_crc_value = 0 + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + cur_pos += size + # END window size loop + + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + assert shawriter.sha(as_hex=False) == sha + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + return True - def info_iter(self): - """ - :return: Iterator over all objects in this pack. The iterator yields - OInfo instances""" - return self._iter_objects(as_stream=False) - - def stream_iter(self): - """ - :return: iterator over all objects in this pack. The iterator yields - OStream instances""" - return self._iter_objects(as_stream=True) - - def collect_streams_at_offset(self, offset): - """ - As the version in the PackFile, but can resolve REF deltas within this pack - For more info, see ``collect_streams`` - - :param offset: offset into the pack file at which the object can be found""" - streams = self._pack.collect_streams(offset) - - # try to resolve the last one if needed. It is assumed to be either - # a REF delta, or a base object, as OFFSET deltas are resolved by the pack - if streams[-1].type_id == REF_DELTA: - stream = streams[-1] - while stream.type_id in delta_types: - if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(stream.delta_info) - if sindex is None: - break - stream = self._pack.stream(self._index.offset(sindex)) - streams.append(stream) - else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who - # would do that ;) ? We can handle it though - stream = self._pack.stream(stream.delta_info) - streams.append(stream) - # END handle ref delta - # END resolve ref streams - # END resolve streams - - return streams - - def collect_streams(self, sha): - """ - As ``PackFile.collect_streams``, but takes a sha instead of an offset. - Additionally, ref_delta streams will be resolved within this pack. - If this is not possible, the stream will be left alone, hence it is adivsed - to check for unresolved ref-deltas and resolve them before attempting to - construct a delta stream. - - :param sha: 20 byte sha1 specifying the object whose related streams you want to collect - :return: list of streams, first being the actual object delta, the last being - a possibly unresolved base object. - :raise BadObject:""" - return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - - @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, - object_count = None, zlib_compression = zlib.Z_BEST_SPEED): - """ - Create a new pack by putting all objects obtained by the object_iterator - into a pack which is written using the pack_write method. - The respective index is produced as well if index_write is not Non. - - :param object_iter: iterator yielding odb output objects - :param pack_write: function to receive strings to write into the pack stream - :param indx_write: if not None, the function writes the index file corresponding - to the pack. - :param object_count: if you can provide the amount of objects in your iteration, - this would be the place to put it. Otherwise we have to pre-iterate and store - all items into a list to get the number, which uses more memory than necessary. - :param zlib_compression: the zlib compression level to use - :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack - and over all contents of the index. If index_write was None, index_binsha will be None - - **Note:** The destination of the write functions is up to the user. It could - be a socket, or a file for instance - - **Note:** writes only undeltified objects""" - objs = object_iter - if not object_count: - if not isinstance(object_iter, (tuple, list)): - objs = list(object_iter) - #END handle list type - object_count = len(objs) - #END handle object - - pack_writer = FlexibleSha1Writer(pack_write) - pwrite = pack_writer.write - ofs = 0 # current offset into the pack file - index = None - wants_index = index_write is not None - - # write header - pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) - ofs += 12 - - if wants_index: - index = IndexWriter() - #END handle index header - - actual_count = 0 - for obj in objs: - actual_count += 1 - crc = 0 - - # object header - hdr = create_pack_object_header(obj.type_id, obj.size) - if index_write: - crc = crc32(hdr) - else: - crc = None - #END handle crc - pwrite(hdr) - - # data stream - zstream = zlib.compressobj(zlib_compression) - ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) - assert(br == obj.size) - if wants_index: - index.append(obj.binsha, crc, ofs) - #END handle index - - ofs += len(hdr) + bw - if actual_count == object_count: - break - #END abort once we are done - #END for each object - - if actual_count != object_count: - raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) - #END count assertion - - # write footer - pack_sha = pack_writer.sha(as_hex = False) - assert len(pack_sha) == 20 - pack_write(pack_sha) - ofs += len(pack_sha) # just for completeness ;) - - index_sha = None - if wants_index: - index_sha = index.write(pack_sha, index_write) - #END handle index - - return pack_sha, index_sha - - @classmethod - def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): - """Create a new on-disk entity comprised of a properly named pack file and a properly named - and corresponding index file. The pack contains all OStream objects contained in object iter. - :param base_dir: directory which is to contain the files - :return: PackEntity instance initialized with the new pack - - **Note:** for more information on the other parameters see the write_pack method""" - pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) - index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) - pack_write = lambda d: os.write(pack_fd, d) - index_write = lambda d: os.write(index_fd, d) - - pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) - os.close(pack_fd) - os.close(index_fd) - - fmt = "pack-%s.%s" - new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) - new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) - os.rename(pack_path, new_pack_path) - os.rename(index_path, new_index_path) - - return cls(new_pack_path) - - - #} END interface + def info_iter(self): + """ + :return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + def collect_streams_at_offset(self, offset): + """ + As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + + @classmethod + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + #END handle list type + object_count = len(objs) + #END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + #END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + crc = 0 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + #END handle crc + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + #END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + #END abort once we are done + #END for each object + + if actual_count != object_count: + raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + #END count assertion + + # write footer + pack_sha = pack_writer.sha(as_hex = False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + + index_sha = None + if wants_index: + index_sha = index.write(pack_sha, index_write) + #END handle index + + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + + **Note:** for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) + + + #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 632213c27..6441b1e1a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -9,684 +9,684 @@ import os from fun import ( - msb_size, - stream_copy, - apply_delta_data, - connect_deltas, - DeltaChunkList, - delta_types - ) + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + DeltaChunkList, + delta_types + ) from util import ( - allocate_memory, - LazyMixin, - make_sha, - write, - close, - zlib - ) + allocate_memory, + LazyMixin, + make_sha, + write, + close, + zlib + ) has_perf_mod = False try: - from _perf import apply_delta as c_apply_delta - has_perf_mod = True + from _perf import apply_delta as c_apply_delta + has_perf_mod = True except ImportError: - pass + pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', - 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', - 'FDStream', 'NullStream') +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams class DecompressMemMapReader(LazyMixin): - """Reads data in chunks from a memory map and decompresses it. The client sees - only the uncompressed data, respective file-like read calls are handling on-demand - buffered decompression accordingly - - A constraint on the total size of bytes is activated, simulating - a logical file within a possibly larger physical memory area - - To read efficiently, you clearly don't want to read individual bytes, instead, - read a few kilobytes at least. - - **Note:** The chunk-size should be carefully selected as it will involve quite a bit - of string copying due to the way the zlib is implemented. Its very wasteful, - hence we try to find a good tradeoff between allocation time and number of - times we actually allocate. An own zlib implementation would be good here - to better support streamed reading - it would only need to keep the mmap - and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', - '_cbr', '_phi') - - max_read_size = 512*1024 # currently unused - - def __init__(self, m, close_on_deletion, size=None): - """Initialize with mmap for stream reading - :param m: must be content data - use new if you have object data and no size""" - self._m = m - self._zip = zlib.decompressobj() - self._buf = None # buffer of decompressed bytes - self._buflen = 0 # length of bytes in buffer - if size is not None: - self._s = size # size of uncompressed data to read in total - self._br = 0 # num uncompressed bytes read - self._cws = 0 # start byte of compression window - self._cwe = 0 # end byte of compression window - self._cbr = 0 # number of compressed bytes read - self._phi = False # is True if we parsed the header info - self._close = close_on_deletion # close the memmap on deletion ? - - def _set_cache_(self, attr): - assert attr == '_s' - # only happens for size, which is a marker to indicate we still - # have to parse the header from the stream - self._parse_header_info() - - def __del__(self): - if self._close: - self._m.close() - # END handle resource freeing - - def _parse_header_info(self): - """If this stream contains object data, parse the header info and skip the - stream to a point where each read will yield object content - - :return: parsed type_string, size""" - # read header - maxb = 512 # should really be enough, cgit uses 8192 I believe - self._s = maxb - hdr = self.read(maxb) - hdrend = hdr.find("\0") - type, size = hdr[:hdrend].split(" ") - size = int(size) - self._s = size - - # adjust internal state to match actual header length that we ignore - # The buffer will be depleted first on future reads - self._br = 0 - hdrend += 1 # count terminating \0 - self._buf = StringIO(hdr[hdrend:]) - self._buflen = len(hdr) - hdrend - - self._phi = True - - return type, size - - #{ Interface - - @classmethod - def new(self, m, close_on_deletion=False): - """Create a new DecompressMemMapReader instance for acting as a read-only stream - This method parses the object header from m and returns the parsed - type and size, as well as the created stream instance. - - :param m: memory map on which to oparate. It must be object data ( header + contents ) - :param close_on_deletion: if True, the memory map will be closed once we are - being deleted""" - inst = DecompressMemMapReader(m, close_on_deletion, 0) - type, size = inst._parse_header_info() - return type, size, inst - - def data(self): - """:return: random access compatible data we are working on""" - return self._m - - def compressed_bytes_read(self): - """ - :return: number of compressed bytes read. This includes the bytes it - took to decompress the header ( if there was one )""" - # ABSTRACT: When decompressing a byte stream, it can be that the first - # x bytes which were requested match the first x bytes in the loosely - # compressed datastream. This is the worst-case assumption that the reader - # does, it assumes that it will get at least X bytes from X compressed bytes - # in call cases. - # The caveat is that the object, according to our known uncompressed size, - # is already complete, but there are still some bytes left in the compressed - # stream that contribute to the amount of compressed bytes. - # How can we know that we are truly done, and have read all bytes we need - # to read ? - # Without help, we cannot know, as we need to obtain the status of the - # decompression. If it is not finished, we need to decompress more data - # until it is finished, to yield the actual number of compressed bytes - # belonging to the decompressed object - # We are using a custom zlib module for this, if its not present, - # we try to put in additional bytes up for decompression if feasible - # and check for the unused_data. - - # Only scrub the stream forward if we are officially done with the - # bytes we were to have. - if self._br == self._s and not self._zip.unused_data: - # manipulate the bytes-read to allow our own read method to coninute - # but keep the window at its current position - self._br = 0 - if hasattr(self._zip, 'status'): - while self._zip.status == zlib.Z_OK: - self.read(mmap.PAGESIZE) - # END scrub-loop custom zlib - else: - # pass in additional pages, until we have unused data - while not self._zip.unused_data and self._cbr != len(self._m): - self.read(mmap.PAGESIZE) - # END scrub-loop default zlib - # END handle stream scrubbing - - # reset bytes read, just to be sure - self._br = self._s - # END handle stream scrubbing - - # unused data ends up in the unconsumed tail, which was removed - # from the count already - return self._cbr - - #} END interface - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Allows to reset the stream to restart reading - :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - - self._zip = zlib.decompressobj() - self._br = self._cws = self._cwe = self._cbr = 0 - if self._phi: - self._phi = False - del(self._s) # trigger header parsing on first access - # END skip header - - def read(self, size=-1): - if size < 1: - size = self._s - self._br - else: - size = min(size, self._s - self._br) - # END clamp size - - if size == 0: - return str() - # END handle depletion - - - # deplete the buffer, then just continue using the decompress object - # which has an own buffer. We just need this to transparently parse the - # header from the zlib stream - dat = str() - if self._buf: - if self._buflen >= size: - # have enough data - dat = self._buf.read(size) - self._buflen -= size - self._br += size - return dat - else: - dat = self._buf.read() # ouch, duplicates data - size -= self._buflen - self._br += self._buflen - - self._buflen = 0 - self._buf = None - # END handle buffer len - # END handle buffer - - # decompress some data - # Abstract: zlib needs to operate on chunks of our memory map ( which may - # be large ), as it will otherwise and always fill in the 'unconsumed_tail' - # attribute which possible reads our whole map to the end, forcing - # everything to be read from disk even though just a portion was requested. - # As this would be a nogo, we workaround it by passing only chunks of data, - # moving the window into the memory map along as we decompress, which keeps - # the tail smaller than our chunk-size. This causes 'only' the chunk to be - # copied once, and another copy of a part of it when it creates the unconsumed - # tail. We have to use it to hand in the appropriate amount of bytes durin g - # the next read. - tail = self._zip.unconsumed_tail - if tail: - # move the window, make it as large as size demands. For code-clarity, - # we just take the chunk from our map again instead of reusing the unconsumed - # tail. The latter one would safe some memory copying, but we could end up - # with not getting enough data uncompressed, so we had to sort that out as well. - # Now we just assume the worst case, hence the data is uncompressed and the window - # needs to be as large as the uncompressed bytes we want to read. - self._cws = self._cwe - len(tail) - self._cwe = self._cws + size - else: - cws = self._cws - self._cws = self._cwe - self._cwe = cws + size - # END handle tail - - - # if window is too small, make it larger so zip can decompress something - if self._cwe - self._cws < 8: - self._cwe = self._cws + 8 - # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) - - # get the actual window end to be sure we don't use it for computations - self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read - # We feed possibly overlapping chunks, which is why the unconsumed tail - # has to be taken into consideration, as well as the unused data - # if we hit the end of the stream - self._cbr += len(indata) - len(self._zip.unconsumed_tail) - self._br += len(dcompdat) - - if dat: - dcompdat = dat + dcompdat - # END prepend our cached data - - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. - # Recursively resolve that. - # Note: dcompdat can be empty even though we still appear to have bytes - # to read, if we are called by compressed_bytes_read - it manipulates - # us to empty the stream - if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size-len(dcompdat)) - # END handle special case - return dcompdat - - + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, thats all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') + + max_read_size = 512*1024 # currently unused + + def __init__(self, m, close_on_deletion, size=None): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + if size is not None: + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info + self._close = close_on_deletion # close the memmap on deletion ? + + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + + def __del__(self): + if self._close: + self._m.close() + # END handle resource freeing + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + + :return: parsed type_string, size""" + # read header + maxb = 512 # should really be enough, cgit uses 8192 I believe + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find("\0") + type, size = hdr[:hdrend].split(" ") + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 # count terminating \0 + self._buf = StringIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + self._phi = True + + return type, size + + #{ Interface + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + + :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + type, size = inst._parse_header_info() + return type, size, inst + + def data(self): + """:return: random access compatible data we are working on""" + return self._m + + def compressed_bytes_read(self): + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and not self._zip.unused_data: + # manipulate the bytes-read to allow our own read method to coninute + # but keep the window at its current position + self._br = 0 + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr + + #} END interface + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return str() + # END handle depletion + + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = str() + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes durin g + # the next read. + tail = self._zip.unconsumed_tail + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + size + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + size + # END handle tail + + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = buffer(self._m, self._cws, self._cwe - self._cws) + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + dcompdat = self._zip.decompress(indata, size) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + self._cbr += len(indata) - len(self._zip.unconsumed_tail) + self._br += len(dcompdat) + + if dat: + dcompdat = dat + dcompdat + # END prepend our cached data + + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. + # Recursively resolve that. + # Note: dcompdat can be empty even though we still appear to have bytes + # to read, if we are called by compressed_bytes_read - it manipulates + # us to empty the stream + if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: + dcompdat += self.read(size-len(dcompdat)) + # END handle special case + return dcompdat + + class DeltaApplyReader(LazyMixin): - """A reader which dynamically applies pack deltas to a base object, keeping the - memory demands to a minimum. - - The size of the final object is only obtainable once all deltas have been - applied, unless it is retrieved from a pack index. - - The uncompressed Delta has the following layout (MSB being a most significant - bit encoded dynamic size): - - * MSB Source Size - the size of the base against which the delta was created - * MSB Target Size - the size of the resulting data after the delta was applied - * A list of one byte commands (cmd) which are followed by a specific protocol: - - * cmd & 0x80 - copy delta_data[offset:offset+size] - - * Followed by an encoded offset into the delta data - * Followed by an encoded size of the chunk to copy - - * cmd & 0x7f - insert - - * insert cmd bytes from the delta buffer into the output stream - - * cmd == 0 - invalid operation ( or error in delta stream ) - """ - __slots__ = ( - "_bstream", # base stream to which to apply the deltas - "_dstreams", # tuple of delta stream readers - "_mm_target", # memory map of the delta-applied data - "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read - ) - - #{ Configuration - k_max_memory_move = 250*1000*1000 - #} END configuration - - def __init__(self, stream_list): - """Initialize this instance with a list of streams, the first stream being - the delta to apply on top of all following deltas, the last stream being the - base object onto which to apply the deltas""" - assert len(stream_list) > 1, "Need at least one delta and one base stream" - - self._bstream = stream_list[-1] - self._dstreams = tuple(stream_list[:-1]) - self._br = 0 - - def _set_cache_too_slow_without_c(self, attr): - # the direct algorithm is fastest and most direct if there is only one - # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python, every function call costs - # huge amounts of time - # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: - if len(self._dstreams) == 1: - return self._set_cache_brute_(attr) - - # Aggregate all deltas into one delta in reverse order. Hence we take - # the last delta, and reverse-merge its ancestor delta, until we receive - # the final delta data stream. - # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) - dcl = connect_deltas(self._dstreams) - - # call len directly, as the (optional) c version doesn't implement the sequence - # protocol - if dcl.rbound() == 0: - self._size = 0 - self._mm_target = allocate_memory(0) - return - # END handle empty list - - self._size = dcl.rbound() - self._mm_target = allocate_memory(self._size) - - bbuf = allocate_memory(self._bstream.size) - stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - - # APPLY CHUNKS - write = self._mm_target.write - dcl.apply(bbuf, write) - - self._mm_target.seek(0) - - def _set_cache_brute_(self, attr): - """If we are here, we apply the actual deltas""" - - # TODO: There should be a special case if there is only one stream - # Then the default-git algorithm should perform a tad faster, as the - # delta is not peaked into, causing less overhead. - buffer_info_list = list() - max_target_size = 0 - for dstream in self._dstreams: - buf = dstream.read(512) # read the header information + X - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) - max_target_size = max(max_target_size, target_size) - # END for each delta stream - - # sanity check - the first delta to apply should have the same source - # size as our actual base stream - base_size = self._bstream.size - target_size = max_target_size - - # if we have more than 1 delta to apply, we will swap buffers, hence we must - # assure that all buffers we use are large enough to hold all the results - if len(self._dstreams) > 1: - base_size = target_size = max(base_size, max_target_size) - # END adjust buffer sizes - - - # Allocate private memory map big enough to hold the first base buffer - # We need random access to it - bbuf = allocate_memory(base_size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) - - # allocate memory map large enough for the largest (intermediate) target - # We will use it as scratch space for all delta ops. If the final - # target buffer is smaller than our allocated space, we just use parts - # of it upon return. - tbuf = allocate_memory(target_size) - - # for each delta to apply, memory map the decompressed delta and - # work on the op-codes to reconstruct everything. - # For the actual copying, we use a seek and write pattern of buffer - # slices. - final_target_size = None - for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): - # allocate a buffer to hold all delta data - fill in the data for - # fast access. We do this as we know that reading individual bytes - # from our stream would be slower than necessary ( although possible ) - # The dbuf buffer contains commands after the first two MSB sizes, the - # offset specifies the amount of bytes read to get the sizes. - ddata = allocate_memory(dstream.size - offset) - ddata.write(dbuf) - # read the rest from the stream. The size we give is larger than necessary - stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - - ####################################################################### - if 'c_apply_delta' in globals(): - c_apply_delta(bbuf, ddata, tbuf); - else: - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) - ####################################################################### - - # finally, swap out source and target buffers. The target is now the - # base for the next delta to apply - bbuf, tbuf = tbuf, bbuf - bbuf.seek(0) - tbuf.seek(0) - final_target_size = target_size - # END for each delta to apply - - # its already seeked to 0, constrain it to the actual size - # NOTE: in the end of the loop, it swaps buffers, hence our target buffer - # is not tbuf, but bbuf ! - self._mm_target = bbuf - self._size = final_target_size - - - #{ Configuration - if not has_perf_mod: - _set_cache_ = _set_cache_brute_ - else: - _set_cache_ = _set_cache_too_slow_without_c - - #} END configuration - - def read(self, count=0): - bl = self._size - self._br # bytes left - if count < 1 or count > bl: - count = bl - # NOTE: we could check for certain size limits, and possibly - # return buffers instead of strings to prevent byte copying - data = self._mm_target.read(count) - self._br += len(data) - return data - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Allows to reset the stream to restart reading - - :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - self._br = 0 - self._mm_target.seek(0) - - #{ Interface - - @classmethod - def new(cls, stream_list): - """ - Convert the given list of streams into a stream which resolves deltas - when reading from it. - - :param stream_list: two or more stream objects, first stream is a Delta - to the object that you want to resolve, followed by N additional delta - streams. The list's last stream must be a non-delta stream. - - :return: Non-Delta OPackStream object whose stream can be used to obtain - the decompressed resolved data - :raise ValueError: if the stream list cannot be handled""" - if len(stream_list) < 2: - raise ValueError("Need at least two streams") - # END single object special handling - - if stream_list[-1].type_id in delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) - # END check stream - - return cls(stream_list) - - #} END interface - - - #{ OInfo like Interface - - @property - def type(self): - return self._bstream.type - - @property - def type_id(self): - return self._bstream.type_id - - @property - def size(self): - """:return: number of uncompressed bytes in the stream""" - return self._size - - #} END oinfo like interface - - + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) + + #{ Configuration + k_max_memory_move = 250*1000*1000 + #} END configuration + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 + + def _set_cache_too_slow_without_c(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) + dcl = connect_deltas(self._dstreams) + + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.rbound() == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl.rbound() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) + + # APPLY CHUNKS + write = self._mm_target.write + dcl.apply(bbuf, write) + + self._mm_target.seek(0) + + def _set_cache_brute_(self, attr): + """If we are here, we apply the actual deltas""" + + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. + buffer_info_list = list() + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + + + # Allocate private memory map big enough to hold the first base buffer + # We need random access to it + bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it upon return. + tbuf = allocate_memory(target_size) + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + + ####################################################################### + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf); + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + ####################################################################### + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + final_target_size = target_size + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size + + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c + + #} END configuration + + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self._br = 0 + self._mm_target.seek(0) + + #{ Interface + + @classmethod + def new(cls, stream_list): + """ + Convert the given list of streams into a stream which resolves deltas + when reading from it. + + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + return cls(stream_list) + + #} END interface + + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + #} END RO streams #{ W Streams class Sha1Writer(object): - """Simple stream writer which produces a sha whenever you like as it degests - everything it is supposed to write""" - __slots__ = "sha1" - - def __init__(self): - self.sha1 = make_sha() - - #{ Stream Interface - - def write(self, data): - """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" - self.sha1.update(data) - return len(data) - - # END stream interface - - #{ Interface - - def sha(self, as_hex = False): - """:return: sha so far - :param as_hex: if True, sha will be hex-encoded, binary otherwise""" - if as_hex: - return self.sha1.hexdigest() - return self.sha1.digest() - - #} END interface + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha() + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex = False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface class FlexibleSha1Writer(Sha1Writer): - """Writer producing a sha1 while passing on the written bytes to the given - write function""" - __slots__ = 'writer' - - def __init__(self, writer): - Sha1Writer.__init__(self) - self.writer = writer - - def write(self, data): - Sha1Writer.write(self, data) - self.writer(data) + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) class ZippedStoreShaWriter(Sha1Writer): - """Remembers everything someone writes to it and generates a sha""" - __slots__ = ('buf', 'zip') - def __init__(self): - Sha1Writer.__init__(self) - self.buf = StringIO() - self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - - def __getattr__(self, attr): - return getattr(self.buf, attr) - - def write(self, data): - alen = Sha1Writer.write(self, data) - self.buf.write(self.zip.compress(data)) - return alen - - def close(self): - self.buf.write(self.zip.flush()) - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Seeking currently only supports to rewind written data - Multiple writes are not supported""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - self.buf.seek(0) - - def getvalue(self): - """:return: string value from the current stream position to the end""" - return self.buf.getvalue() + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() class FDCompressedSha1Writer(Sha1Writer): - """Digests data written to it, making the sha available, then compress the - data and write it to the file descriptor - - **Note:** operates on raw file descriptors - **Note:** for this to work, you have to use the close-method of this instance""" - __slots__ = ("fd", "sha1", "zip") - - # default exception - exc = IOError("Failed to write all bytes to filedescriptor") - - def __init__(self, fd): - super(FDCompressedSha1Writer, self).__init__() - self.fd = fd - self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - - #{ Stream Interface - - def write(self, data): - """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" - self.sha1.update(data) - cdata = self.zip.compress(data) - bytes_written = write(self.fd, cdata) - if bytes_written != len(cdata): - raise self.exc - return len(data) - - def close(self): - remainder = self.zip.flush() - if write(self.fd, remainder) != len(remainder): - raise self.exc - return close(self.fd) - - #} END stream interface + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super(FDCompressedSha1Writer, self).__init__() + self.fd = fd + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): + raise self.exc + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface class FDStream(object): - """A simple wrapper providing the most basic functions on a file descriptor - with the fileobject interface. Cannot use os.fdopen as the resulting stream - takes ownership""" - __slots__ = ("_fd", '_pos') - def __init__(self, fd): - self._fd = fd - self._pos = 0 - - def write(self, data): - self._pos += len(data) - os.write(self._fd, data) - - def read(self, count=0): - if count == 0: - count = os.path.getsize(self._filepath) - # END handle read everything - - bytes = os.read(self._fd, count) - self._pos += len(bytes) - return bytes - - def fileno(self): - return self._fd - - def tell(self): - return self._pos - - def close(self): - close(self._fd) + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + def close(self): + close(self._fd) class NullStream(object): - """A stream that does nothing but providing a stream interface. - Use it like /dev/null""" - __slots__ = tuple() - - def read(self, size=0): - return '' - - def close(self): - pass - - def write(self, data): - return len(data) + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) #} END W streams diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 760f531be..f8059447f 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -7,10 +7,10 @@ #{ Initialization def _init_pool(): - """Assure the pool is actually threaded""" - size = 2 - print "Setting ThreadPool to %i" % size - gitdb.util.pool.set_size(size) + """Assure the pool is actually threaded""" + size = 2 + print "Setting ThreadPool to %i" % size + gitdb.util.pool.set_size(size) #} END initialization diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 4af4483c7..62614ee5c 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -4,21 +4,21 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Base classes for object db testing""" from gitdb.test.lib import ( - with_rw_directory, - with_packs_rw, - ZippedStoreShaWriter, - fixture_path, - TestBase - ) + with_rw_directory, + with_packs_rw, + ZippedStoreShaWriter, + fixture_path, + TestBase + ) from gitdb.stream import Sha1Writer from gitdb.base import ( - IStream, - OStream, - OInfo - ) - + IStream, + OStream, + OInfo + ) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type @@ -28,181 +28,181 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') - + class TestDBBase(TestBase): - """Base class providing testing routines on databases""" - - # data - two_lines = "1234\nhello world" - all_data = (two_lines, ) - - def _assert_object_writing_simple(self, db): - # write a bunch of objects and query their streams and info - null_objs = db.size() - ni = 250 - for i in xrange(ni): - data = pack(">L", i) - istream = IStream(str_blob_type, len(data), StringIO(data)) - new_istream = db.store(istream) - assert new_istream is istream - assert db.has_object(istream.binsha) - - info = db.info(istream.binsha) - assert isinstance(info, OInfo) - assert info.type == istream.type and info.size == istream.size - - stream = db.stream(istream.binsha) - assert isinstance(stream, OStream) - assert stream.binsha == info.binsha and stream.type == info.type - assert stream.read() == data - # END for each item - - assert db.size() == null_objs + ni - shas = list(db.sha_iter()) - assert len(shas) == db.size() - assert len(shas[0]) == 20 - - - def _assert_object_writing(self, db): - """General tests to verify object writing, compatible to ObjectDBW - **Note:** requires write access to the database""" - # start in 'dry-run' mode, using a simple sha1 writer - ostreams = (ZippedStoreShaWriter, None) - for ostreamcls in ostreams: - for data in self.all_data: - dry_run = ostreamcls is not None - ostream = None - if ostreamcls is not None: - ostream = ostreamcls() - assert isinstance(ostream, Sha1Writer) - # END create ostream - - prev_ostream = db.set_ostream(ostream) - assert type(prev_ostream) in ostreams or prev_ostream in ostreams - - istream = IStream(str_blob_type, len(data), StringIO(data)) - - # store returns same istream instance, with new sha set - my_istream = db.store(istream) - sha = istream.binsha - assert my_istream is istream - assert db.has_object(sha) != dry_run - assert len(sha) == 20 - - # verify data - the slow way, we want to run code - if not dry_run: - info = db.info(sha) - assert str_blob_type == info.type - assert info.size == len(data) - - ostream = db.stream(sha) - assert ostream.read() == data - assert ostream.type == str_blob_type - assert ostream.size == len(data) - else: - self.failUnlessRaises(BadObject, db.info, sha) - self.failUnlessRaises(BadObject, db.stream, sha) - - # DIRECT STREAM COPY - # our data hase been written in object format to the StringIO - # we pasesd as output stream. No physical database representation - # was created. - # Test direct stream copy of object streams, the result must be - # identical to what we fed in - ostream.seek(0) - istream.stream = ostream - assert istream.binsha is not None - prev_sha = istream.binsha - - db.set_ostream(ZippedStoreShaWriter()) - db.store(istream) - assert istream.binsha == prev_sha - new_ostream = db.ostream() - - # note: only works as long our store write uses the same compression - # level, which is zip_best - assert ostream.getvalue() == new_ostream.getvalue() - # END for each data set - # END for each dry_run mode - - def _assert_object_writing_async(self, db): - """Test generic object writing using asynchronous access""" - ni = 5000 - def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - data = str(data_src + offset) - yield IStream(str_blob_type, len(data), StringIO(data)) - # END for each item - # END generator utility - - # for now, we are very trusty here as we expect it to work if it worked - # in the single-stream case - - # write objects - reader = IteratorReader(istream_generator()) - istream_reader = db.store_async(reader) - istreams = istream_reader.read() # read all - assert istream_reader.task().error() is None - assert len(istreams) == ni - - for stream in istreams: - assert stream.error is None - assert len(stream.binsha) == 20 - assert isinstance(stream, IStream) - # END assert each stream - - # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.binsha for istream in istreams ) - hasobject_reader = db.has_object_async(reader) - count = 0 - for sha, has_object in hasobject_reader: - assert has_object - count += 1 - # END for each sha - assert count == ni - - # read the objects we have just written - reader = IteratorReader( istream.binsha for istream in istreams ) - ostream_reader = db.stream_async(reader) - - # read items individually to prevent hitting possible sys-limits - count = 0 - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert ostream_reader.task().error() is None - assert count == ni - - # get info about our items - reader = IteratorReader( istream.binsha for istream in istreams ) - info_reader = db.info_async(reader) - - count = 0 - for oinfo in info_reader: - assert isinstance(oinfo, OInfo) - count += 1 - # END for each oinfo instance - assert count == ni - - - # combined read-write using a converter - # add 2500 items, and obtain their output streams - nni = 2500 - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - - istream_reader = db.store_async(reader) - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = db.stream_async(istream_reader) - - count = 0 - # read it individually, otherwise we might run into the ulimit - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert count == nni - - + """Base class providing testing routines on databases""" + + # data + two_lines = "1234\nhello world" + all_data = (two_lines, ) + + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in xrange(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), StringIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.binsha) + + info = db.info(istream.binsha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.binsha) + assert isinstance(stream, OStream) + assert stream.binsha == info.binsha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + **Note:** requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + + istream = IStream(str_blob_type, len(data), StringIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.binsha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 20 + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert str_blob_type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == str_blob_type + assert ostream.size == len(data) + else: + self.failUnlessRaises(BadObject, db.info, sha) + self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.binsha is not None + prev_sha = istream.binsha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.binsha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip_best + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode + + def _assert_object_writing_async(self, db): + """Test generic object writing using asynchronous access""" + ni = 5000 + def istream_generator(offset=0, ni=ni): + for data_src in xrange(ni): + data = str(data_src + offset) + yield IStream(str_blob_type, len(data), StringIO(data)) + # END for each item + # END generator utility + + # for now, we are very trusty here as we expect it to work if it worked + # in the single-stream case + + # write objects + reader = IteratorReader(istream_generator()) + istream_reader = db.store_async(reader) + istreams = istream_reader.read() # read all + assert istream_reader.task().error() is None + assert len(istreams) == ni + + for stream in istreams: + assert stream.error is None + assert len(stream.binsha) == 20 + assert isinstance(stream, IStream) + # END assert each stream + + # test has-object-async - we must have all previously added ones + reader = IteratorReader( istream.binsha for istream in istreams ) + hasobject_reader = db.has_object_async(reader) + count = 0 + for sha, has_object in hasobject_reader: + assert has_object + count += 1 + # END for each sha + assert count == ni + + # read the objects we have just written + reader = IteratorReader( istream.binsha for istream in istreams ) + ostream_reader = db.stream_async(reader) + + # read items individually to prevent hitting possible sys-limits + count = 0 + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert ostream_reader.task().error() is None + assert count == ni + + # get info about our items + reader = IteratorReader( istream.binsha for istream in istreams ) + info_reader = db.info_async(reader) + + count = 0 + for oinfo in info_reader: + assert isinstance(oinfo, OInfo) + count += 1 + # END for each oinfo instance + assert count == ni + + + # combined read-write using a converter + # add 2500 items, and obtain their output streams + nni = 2500 + reader = IteratorReader(istream_generator(offset=ni, ni=nni)) + istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] + + istream_reader = db.store_async(reader) + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = db.stream_async(istream_reader) + + count = 0 + # read it individually, otherwise we might run into the ulimit + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert count == nni + + diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 310116351..1ef577aa3 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,41 +7,41 @@ from gitdb.db import GitDB from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex - + class TestGitDB(TestDBBase): - - def test_reading(self): - gdb = GitDB(fixture_path('../../../.git/objects')) - - # we have packs and loose objects, alternates doesn't necessarily exist - assert 1 < len(gdb.databases()) < 4 - - # access should be possible - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") - assert isinstance(gdb.info(gitdb_sha), OInfo) - assert isinstance(gdb.stream(gitdb_sha), OStream) - assert gdb.size() > 200 - sha_list = list(gdb.sha_iter()) - assert len(sha_list) == gdb.size() - - - # This is actually a test for compound functionality, but it doesn't - # have a separate test module - # test partial shas - # this one as uneven and quite short - assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") - - # mix even/uneven hexshas - for i, binsha in enumerate(sha_list): - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha - # END for each sha - - self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") - - @with_rw_directory - def test_writing(self, path): - gdb = GitDB(path) - - # its possible to write objects - self._assert_object_writing(gdb) - self._assert_object_writing_async(gdb) + + def test_reading(self): + gdb = GitDB(fixture_path('../../../.git/objects')) + + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + assert gdb.size() > 200 + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module + # test partial shas + # this one as uneven and quite short + assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha + # END for each sha + + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) + self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index ee2d78d08..d7e1d01b0 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -6,29 +6,29 @@ from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex - + class TestLooseDB(TestDBBase): - - @with_rw_directory - def test_basics(self, path): - ldb = LooseObjectDB(path) - - # write data - self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) - - # verify sha iteration and size - shas = list(ldb.sha_iter()) - assert shas and len(shas[0]) == 20 - - assert len(shas) == ldb.size() - - # verify find short object - long_sha = bin_to_hex(shas[-1]) - for short_sha in (long_sha[:20], long_sha[:5]): - assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha - # END for each sha - - self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') - # raises if no object could be foudn - + + @with_rw_directory + def test_basics(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) + + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be foudn + diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 188cb0a93..df428e2b7 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -4,27 +4,27 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( - MemoryDB, - LooseObjectDB - ) - + MemoryDB, + LooseObjectDB + ) + class TestMemoryDB(TestDBBase): - - @with_rw_directory - def test_writing(self, path): - mdb = MemoryDB() - - # write data - self._assert_object_writing_simple(mdb) - - # test stream copy - ldb = LooseObjectDB(path) - assert ldb.size() == 0 - num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) - assert num_streams_copied == mdb.size() - - assert ldb.size() == mdb.size() - for sha in mdb.sha_iter(): - assert ldb.has_object(sha) - assert ldb.stream(sha).read() == mdb.stream(sha).read() - # END verify objects where copied and are equal + + @with_rw_directory + def test_writing(self, path): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index e8ba6f8fc..f4cb5bbc6 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -12,62 +12,62 @@ import random class TestPackDB(TestDBBase): - - @with_rw_directory - @with_packs_rw - def test_writing(self, path): - pdb = PackedDB(path) - - # on demand, we init our pack cache - num_packs = len(pdb.entities()) - assert pdb._st_mtime != 0 - - # test pack directory changed: - # packs removed - rename a file, should affect the glob - pack_path = pdb.entities()[0].pack().path() - new_pack_path = pack_path + "renamed" - os.rename(pack_path, new_pack_path) - - pdb.update_cache(force=True) - assert len(pdb.entities()) == num_packs - 1 - - # packs added - os.rename(new_pack_path, pack_path) - pdb.update_cache(force=True) - assert len(pdb.entities()) == num_packs - - # bang on the cache - # access the Entities directly, as there is no iteration interface - # yet ( or required for now ) - sha_list = list(pdb.sha_iter()) - assert len(sha_list) == pdb.size() - - # hit all packs in random order - random.shuffle(sha_list) - - for sha in sha_list: - info = pdb.info(sha) - stream = pdb.stream(sha) - # END for each sha to query - - - # test short finding - be a bit more brutal here - max_bytes = 19 - min_bytes = 2 - num_ambiguous = 0 - for i, sha in enumerate(sha_list): - short_sha = sha[:max((i % max_bytes), min_bytes)] - try: - assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha - except AmbiguousObjectName: - num_ambiguous += 1 - pass # valid, we can have short objects - # END exception handling - # END for each sha to find - - # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... - # assert num_ambiguous - - # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) + + @with_rw_directory + @with_packs_rw + def test_writing(self, path): + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = len(pdb.entities()) + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb.entities()[0].pack().path() + new_pack_path = pack_path + "renamed" + os.rename(pack_path, new_pack_path) + + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs + + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + info = pdb.info(sha) + stream = pdb.stream(sha) + # END for each sha to query + + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambigious ... + # assert num_ambiguous + + # non-existing + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 0d8eeebb3..1637bff74 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -6,55 +6,55 @@ from gitdb.db import ReferenceDB from gitdb.util import ( - NULL_BIN_SHA, - hex_to_bin - ) + NULL_BIN_SHA, + hex_to_bin + ) import os - + class TestReferenceDB(TestDBBase): - - def make_alt_file(self, alt_path, alt_list): - """Create an alternates file which contains the given alternates. - The list can be empty""" - alt_file = open(alt_path, "wb") - for alt in alt_list: - alt_file.write(alt + "\n") - alt_file.close() - - @with_rw_directory - def test_writing(self, path): - NULL_BIN_SHA = '\0' * 20 - - alt_path = os.path.join(path, 'alternates') - rdb = ReferenceDB(alt_path) - assert len(rdb.databases()) == 0 - assert rdb.size() == 0 - assert len(list(rdb.sha_iter())) == 0 - - # try empty, non-existing - assert not rdb.has_object(NULL_BIN_SHA) - - - # setup alternate file - # add two, one is invalid - own_repo_path = fixture_path('../../../.git/objects') # use own repo - self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) - rdb.update_cache() - assert len(rdb.databases()) == 1 - - # we should now find a default revision of ours - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") - assert rdb.has_object(gitdb_sha) - - # remove valid - self.make_alt_file(alt_path, ["just/one/invalid/path"]) - rdb.update_cache() - assert len(rdb.databases()) == 0 - - # add valid - self.make_alt_file(alt_path, [own_repo_path]) - rdb.update_cache() - assert len(rdb.databases()) == 1 - - + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + alt_file = open(alt_path, "wb") + for alt in alt_list: + alt_file.write(alt + "\n") + alt_file.close() + + @with_rw_directory + def test_writing(self, path): + NULL_BIN_SHA = '\0' * 20 + + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 + + # try empty, non-existing + assert not rdb.has_object(NULL_BIN_SHA) + + + # setup alternate file + # add two, one is invalid + own_repo_path = fixture_path('../../../.git/objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + assert rdb.has_object(gitdb_sha) + + # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 + + # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 50645be65..ac8473a4e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,12 +4,12 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( - OStream, - ) + OStream, + ) from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter - ) + Sha1Writer, + ZippedStoreShaWriter + ) from gitdb.util import zlib @@ -29,134 +29,134 @@ #{ Bases class TestBase(unittest.TestCase): - """Base class for all tests""" - + """Base class for all tests""" + #} END bases #{ Decorators def with_rw_directory(func): - """Create a temporary directory which can be written to, remove it if the - test suceeds, but leave it otherwise to aid additional debugging""" - def wrapper(self): - path = tempfile.mktemp(prefix=func.__name__) - os.mkdir(path) - keep = False - try: - try: - return func(self, path) - except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) - keep = True - raise - finally: - # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as - # memory maps closed, once objects go out of scope. For some reason - # though this is not the case here unless we collect explicitly. - if not keep: - gc.collect() - shutil.rmtree(path) - # END handle exception - # END wrapper - - wrapper.__name__ = func.__name__ - return wrapper + """Create a temporary directory which can be written to, remove it if the + test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): + path = tempfile.mktemp(prefix=func.__name__) + os.mkdir(path) + keep = False + try: + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + keep = True + raise + finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + if not keep: + gc.collect() + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper def with_packs_rw(func): - """Function that provides a path into which the packs for testing should be - copied. Will pass on the path to the actual function afterwards""" - def wrapper(self, path): - src_pack_glob = fixture_path('packs/*') - copy_files_globbed(src_pack_glob, path, hard_link_ok=True) - return func(self, path) - # END wrapper - - wrapper.__name__ = func.__name__ - return wrapper + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper #} END decorators #{ Routines def fixture_path(relapath=''): - """:return: absolute path into the fixture directory - :param relapath: relative path into the fixtures directory, or '' - to obtain the fixture directory itself""" - return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) - + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): - """Copy all files found according to the given source glob into the target directory - :param hard_link_ok: if True, hard links will be created if possible. Otherwise - the files will be copied""" - for src_file in glob.glob(source_glob): - if hard_link_ok and hasattr(os, 'link'): - target = os.path.join(target_dir, os.path.basename(src_file)) - try: - os.link(src_file, target) - except OSError: - shutil.copy(src_file, target_dir) - # END handle cross device links ( and resulting failure ) - else: - shutil.copy(src_file, target_dir) - # END try hard link - # END for each file to copy - + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + def make_bytes(size_in_bytes, randomize=False): - """:return: string with given size in bytes - :param randomize: try to produce a very random stream""" - actual_size = size_in_bytes / 4 - producer = xrange(actual_size) - if randomize: - producer = list(producer) - random.shuffle(producer) - # END randomize - a = array('i', producer) - return a.tostring() + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tostring() def make_object(type, data): - """:return: bytes resembling an uncompressed object""" - odata = "blob %i\0" % len(data) - return odata + data - + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata + data + def make_memory_file(size_in_bytes, randomize=False): - """:return: tuple(size_of_stream, stream) - :param randomize: try to produce a very random stream""" - d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) #} END routines #{ Stream Utilities class DummyStream(object): - def __init__(self): - self.was_read = False - self.bytes = 0 - self.closed = False - - def read(self, size): - self.was_read = True - self.bytes = size - - def close(self): - self.closed = True - - def _assert(self): - assert self.was_read + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read class DeriveTest(OStream): - def __init__(self, sha, type, size, stream, *args, **kwargs): - self.myarg = kwargs.pop('myarg') - self.args = args - - def _assert(self): - assert self.args - assert self.myarg + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg #} END stream utilitiess diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 761113d51..3563fcfbe 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -16,12 +16,12 @@ #{ Utilities def resolve_or_fail(env_var): - """:return: resolved environment variable or raise EnvironmentError""" - try: - return os.environ[env_var] - except KeyError: - raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) - # END exception handling + """:return: resolved environment variable or raise EnvironmentError""" + try: + return os.environ[env_var] + except KeyError: + raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) + # END exception handling #} END utilities @@ -29,26 +29,26 @@ def resolve_or_fail(env_var): #{ Base Classes class TestBigRepoR(TestBase): - """TestCase providing access to readonly 'big' repositories using the following - member variables: - - * gitrepopath - - * read-only base path of the git source repository, i.e. .../git/.git""" - - #{ Invariants - head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' - head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' - #} END invariants - - @classmethod - def setUpAll(cls): - try: - super(TestBigRepoR, cls).setUpAll() - except AttributeError: - pass - cls.gitrepopath = resolve_or_fail(k_env_git_repo) - assert cls.gitrepopath.endswith('.git') - - + """TestCase providing access to readonly 'big' repositories using the following + member variables: + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git""" + + #{ Invariants + head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' + head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' + #} END invariants + + @classmethod + def setUpAll(cls): + try: + super(TestBigRepoR, cls).setUpAll() + except AttributeError: + pass + cls.gitrepopath = resolve_or_fail(k_env_git_repo) + assert cls.gitrepopath.endswith('.git') + + #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 20618024d..63856e218 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -4,8 +4,8 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" from lib import ( - TestBigRepoR - ) + TestBigRepoR + ) from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB @@ -18,76 +18,76 @@ from nose import SkipTest class TestPackedDBPerformance(TestBigRepoR): - - def test_pack_random_access(self): - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - # sha lookup - st = time() - sha_list = list(pdb.sha_iter()) - elapsed = time() - st - ns = len(sha_list) - print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) - - # sha lookup: best-case and worst case access - pdb_pack_info = pdb._pack_info - # END shuffle shas - st = time() - for sha in sha_list: - pdb_pack_info(sha) - # END for each sha to look up - elapsed = time() - st - - # discard cache - del(pdb._entities) - pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) - # END for each random mode - - # query info and streams only - max_items = 10000 # can wait longer when testing memory - for pdb_fun in (pdb.info, pdb.stream): - st = time() - for sha in sha_list[:max_items]: - pdb_fun(sha) - elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) - # END for each function - - # retrieve stream and read all - max_items = 5000 - pdb_stream = pdb.stream - total_size = 0 - st = time() - for sha in sha_list[:max_items]: - stream = pdb_stream(sha) - stream.read() - total_size += stream.size - elapsed = time() - st - total_kib = total_size / 1000 - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - - def test_correctness(self): - raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - # disabled for now as it used to work perfectly, checking big repositories takes a long time - print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" - for crc in range(2): - count = 0 - st = time() - for entity in pdb.entities(): - pack_verify = entity.is_valid_stream - sha_by_index = entity.index().sha - for index in xrange(entity.index().size()): - try: - assert pack_verify(sha_by_index(index), use_crc=crc) - count += 1 - except UnsupportedOperation: - pass - # END ignore old indices - # END for each index - # END for each entity - elapsed = time() - st - print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) - # END for each verify mode - + + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + + # discard cache + del(pdb._entities) + pdb.entities() + print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + # END for each random mode + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + # END for each function + + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + + def test_correctness(self): + raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in xrange(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 + except UnsupportedOperation: + pass + # END ignore old indices + # END for each index + # END for each entity + elapsed = time() - st + print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + # END for each verify mode + diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 3c40ed0fb..c66e60cba 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -4,8 +4,8 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" from lib import ( - TestBigRepoR - ) + TestBigRepoR + ) from gitdb.db.pack import PackedDB from gitdb.stream import NullStream @@ -17,63 +17,63 @@ from nose import SkipTest class CountedNullStream(NullStream): - __slots__ = '_bw' - def __init__(self): - self._bw = 0 - - def bytes_written(self): - return self._bw - - def write(self, d): - self._bw += NullStream.write(self, d) - + __slots__ = '_bw' + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + class TestPackStreamingPerformance(TestBigRepoR): - - def test_pack_writing(self): - # see how fast we can write a pack from object streams. - # This will not be fast, as we take time for decompressing the streams as well - ostream = CountedNullStream() - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - ni = 5000 - count = 0 - total_size = 0 - st = time() - for sha in pdb.sha_iter(): - count += 1 - pdb.stream(sha) - if count == ni: - break - #END gather objects for pack-writing - elapsed = time() - st - print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) - - st = time() - PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) - elapsed = time() - st - total_kb = ostream.bytes_written() / 1000 - print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) - - - def test_stream_reading(self): - raise SkipTest() - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - # streaming only, meant for --with-profile runs - ni = 5000 - count = 0 - pdb_stream = pdb.stream - total_size = 0 - st = time() - for sha in pdb.sha_iter(): - if count == ni: - break - stream = pdb_stream(sha) - stream.read() - total_size += stream.size - count += 1 - elapsed = time() - st - total_kib = total_size / 1000 - print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) - + + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 5000 + count = 0 + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + count += 1 + pdb.stream(sha) + if count == ni: + break + #END gather objects for pack-writing + elapsed = time() - st + print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + + st = time() + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + + + def test_stream_reading(self): + raise SkipTest() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index f5f2e2e4d..010003d4b 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -8,16 +8,16 @@ from gitdb.base import * from gitdb.stream import * from gitdb.util import ( - pool, - bin_to_hex - ) + pool, + bin_to_hex + ) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size from async import ( - IteratorReader, - ChannelThreadTask, - ) + IteratorReader, + ChannelThreadTask, + ) from cStringIO import StringIO from time import time @@ -28,168 +28,168 @@ from lib import ( - TestBigRepoR, - make_memory_file, - with_rw_directory - ) + TestBigRepoR, + make_memory_file, + with_rw_directory + ) #{ Utilities def read_chunked_stream(stream): - total = 0 - while True: - chunk = stream.read(chunk_size) - total += len(chunk) - if len(chunk) < chunk_size: - break - # END read stream loop - assert total == stream.size - return stream - - + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + class TestStreamReader(ChannelThreadTask): - """Expects input streams and reads them in chunks. It will read one at a time, - requireing a queue chunk of size 1""" - def __init__(self, *args): - super(TestStreamReader, self).__init__(*args) - self.fun = read_chunked_stream - self.max_chunksize = 1 - + """Expects input streams and reads them in chunks. It will read one at a time, + requireing a queue chunk of size 1""" + def __init__(self, *args): + super(TestStreamReader, self).__init__(*args) + self.fun = read_chunked_stream + self.max_chunksize = 1 + #} END utilities class TestObjDBPerformance(TestBigRepoR): - - large_data_size_bytes = 1000*1000*50 # some MiB should do it - moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - - @with_rw_directory - def test_large_data_streaming(self, path): - ldb = LooseObjectDB(path) - string_ios = list() # list of streams we previously created - - # serial mode - for randomize in range(2): - desc = (randomize and 'random ') or '' - print >> sys.stderr, "Creating %s data ..." % desc - st = time() - size, stream = make_memory_file(self.large_data_size_bytes, randomize) - elapsed = time() - st - print >> sys.stderr, "Done (in %f s)" % elapsed - string_ios.append(stream) - - # writing - due to the compression it will seem faster than it is - st = time() - sha = ldb.store(IStream('blob', size, stream)).binsha - elapsed_add = time() - st - assert ldb.has_object(sha) - db_file = ldb.readable_db_object_path(bin_to_hex(sha)) - fsize_kib = os.path.getsize(db_file) / 1000 - - - size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) - - # reading all at once - st = time() - ostream = ldb.stream(sha) - shadata = ostream.read() - elapsed_readall = time() - st - - stream.seek(0) - assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) - - - # reading in chunks of 1 MiB - cs = 512*1000 - chunks = list() - st = time() - ostream = ldb.stream(sha) - while True: - data = ostream.read(cs) - chunks.append(data) - if len(data) < cs: - break - # END read in chunks - elapsed_readchunks = time() - st - - stream.seek(0) - assert ''.join(chunks) == stream.getvalue() - - cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) - - # del db file so we keep something to do - os.remove(db_file) - # END for each randomization factor - - - # multi-threaded mode - # want two, should be supported by most of todays cpus - pool.set_size(2) - total_kib = 0 - nsios = len(string_ios) - for stream in string_ios: - stream.seek(0) - total_kib += len(stream.getvalue()) / 1000 - # END rewind - - def istream_iter(): - for stream in string_ios: - stream.seek(0) - yield IStream(str_blob_type, len(stream.getvalue()), stream) - # END for each stream - # END util - - # write multiple objects at once, involving concurrent compression - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - st = time() - istreams = istream_reader.read(nsios) - assert len(istreams) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # decompress multiple at once, by reading them - # chunk size is not important as the stream will not really be decompressed - - # until its read - istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.task().max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # store the files, and read them back. For the reading, we use a task - # as well which is chunked into one item per task. Reading all will - # very quickly result in two threads handling two bytestreams of - # chained compression/decompression streams - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - istream_to_sha = lambda items: [ i.binsha for i in items ] - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + large_data_size_bytes = 1000*1000*50 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + + # serial mode + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + string_ios.append(stream) + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).binsha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) + fsize_kib = os.path.getsize(db_file) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so we keep something to do + os.remove(db_file) + # END for each randomization factor + + + # multi-threaded mode + # want two, should be supported by most of todays cpus + pool.set_size(2) + total_kib = 0 + nsios = len(string_ios) + for stream in string_ios: + stream.seek(0) + total_kib += len(stream.getvalue()) / 1000 + # END rewind + + def istream_iter(): + for stream in string_ios: + stream.seek(0) + yield IStream(str_blob_type, len(stream.getvalue()), stream) + # END for each stream + # END util + + # write multiple objects at once, involving concurrent compression + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + st = time() + istreams = istream_reader.read(nsios) + assert len(istreams) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # decompress multiple at once, by reading them + # chunk size is not important as the stream will not really be decompressed + + # until its read + istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + output_reader.task().max_chunksize = 1 + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # store the files, and read them back. For the reading, we use a task + # as well which is chunked into one item per task. Reading all will + # very quickly result in two threads handling two bytestreams of + # chained compression/decompression streams + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + istream_to_sha = lambda items: [ i.binsha for i in items ] + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + output_reader.max_chunksize = 1 + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 1b20faf87..d4ce428c3 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -4,95 +4,95 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( - TestBase, - DummyStream, - DeriveTest, - ) + TestBase, + DummyStream, + DeriveTest, + ) from gitdb import * from gitdb.util import ( - NULL_BIN_SHA - ) + NULL_BIN_SHA + ) from gitdb.typ import ( - str_blob_type - ) + str_blob_type + ) class TestBaseTypes(TestBase): - - def test_streams(self): - # test info - sha = NULL_BIN_SHA - s = 20 - blob_id = 3 - - info = OInfo(sha, str_blob_type, s) - assert info.binsha == sha - assert info.type == str_blob_type - assert info.type_id == blob_id - assert info.size == s - - # test pack info - # provides type_id - pinfo = OPackInfo(0, blob_id, s) - assert pinfo.type == str_blob_type - assert pinfo.type_id == blob_id - assert pinfo.pack_offset == 0 - - dpinfo = ODeltaPackInfo(0, blob_id, s, sha) - assert dpinfo.type == str_blob_type - assert dpinfo.type_id == blob_id - assert dpinfo.delta_info == sha - assert dpinfo.pack_offset == 0 - - - # test ostream - stream = DummyStream() - ostream = OStream(*(info + (stream, ))) - assert ostream.stream is stream - ostream.read(15) - stream._assert() - assert stream.bytes == 15 - ostream.read(20) - assert stream.bytes == 20 - - # test packstream - postream = OPackStream(*(pinfo + (stream, ))) - assert postream.stream is stream - postream.read(10) - stream._assert() - assert stream.bytes == 10 - - # test deltapackstream - dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream - dpostream.read(5) - stream._assert() - assert stream.bytes == 5 - - # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - - # test istream - istream = IStream(str_blob_type, s, stream) - assert istream.binsha == None - istream.binsha = sha - assert istream.binsha == sha - - assert len(istream.binsha) == 20 - assert len(istream.hexsha) == 40 - - assert istream.size == s - istream.size = s * 2 - istream.size == s * 2 - assert istream.type == str_blob_type - istream.type = "something" - assert istream.type == "something" - assert istream.stream is stream - istream.stream = None - assert istream.stream is None - - assert istream.error is None - istream.error = Exception() - assert isinstance(istream.error, Exception) + + def test_streams(self): + # test info + sha = NULL_BIN_SHA + s = 20 + blob_id = 3 + + info = OInfo(sha, str_blob_type, s) + assert info.binsha == sha + assert info.type == str_blob_type + assert info.type_id == blob_id + assert info.size == s + + # test pack info + # provides type_id + pinfo = OPackInfo(0, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 753177560..611ae4299 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -7,58 +7,58 @@ from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool - + from cStringIO import StringIO from async import IteratorReader - + class TestExamples(TestBase): - - def test_base(self): - ldb = LooseObjectDB(fixture_path("../../../.git/objects")) - - for sha1 in ldb.sha_iter(): - oinfo = ldb.info(sha1) - ostream = ldb.stream(sha1) - assert oinfo[:3] == ostream[:3] - - assert len(ostream.read()) == ostream.size - assert ldb.has_object(oinfo.binsha) - # END for each sha in database - # assure we close all files - try: - del(ostream) - del(oinfo) - except UnboundLocalError: - pass - # END ignore exception if there are no loose objects - - data = "my data" - istream = IStream("blob", len(data), StringIO(data)) - - # the object does not yet have a sha - assert istream.binsha is None - ldb.store(istream) - # now the sha is set - assert len(istream.binsha) == 20 - assert ldb.has_object(istream.binsha) - - - # async operation - # Create a reader from an iterator - reader = IteratorReader(ldb.sha_iter()) - - # get reader for object streams - info_reader = ldb.stream_async(reader) - - # read one - info = info_reader.read(1)[0] - - # read all the rest until depletion - ostreams = info_reader.read() - - # set the pool to use two threads - pool.set_size(2) - - # synchronize the mode of operation - pool.set_size(0) + + def test_base(self): + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + # assure we close all files + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + + + # async operation + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 4a7f1caf2..779155a2a 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -4,23 +4,23 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" from lib import ( - TestBase, - with_rw_directory, - with_packs_rw, - fixture_path - ) + TestBase, + with_rw_directory, + with_packs_rw, + fixture_path + ) from gitdb.stream import DeltaApplyReader from gitdb.pack import ( - PackEntity, - PackIndexFile, - PackFile - ) + PackEntity, + PackIndexFile, + PackFile + ) from gitdb.base import ( - OInfo, - OStream, - ) + OInfo, + OStream, + ) from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation @@ -35,213 +35,213 @@ #{ Utilities def bin_sha_from_filename(filename): - return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) #} END utilities class TestPack(TestBase): - - packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) - packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) - packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) - packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) - packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) - packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - - - def _assert_index_file(self, index, version, size): - assert index.packfile_checksum() != index.indexfile_checksum() - assert len(index.packfile_checksum()) == 20 - assert len(index.indexfile_checksum()) == 20 - assert index.version() == version - assert index.size() == size - assert len(index.offsets()) == size - - # get all data of all objects - for oidx in xrange(index.size()): - sha = index.sha(oidx) - assert oidx == index.sha_to_index(sha) - - entry = index.entry(oidx) - assert len(entry) == 3 - - assert entry[0] == index.offset(oidx) - assert entry[1] == sha - assert entry[2] == index.crc(oidx) - - # verify partial sha - for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l], l*2) == oidx - - # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - - - def _assert_pack_file(self, pack, version, size): - assert pack.version() == 2 - assert pack.size() == size - assert len(pack.checksum()) == 20 - - num_obj = 0 - for obj in pack.stream_iter(): - num_obj += 1 - info = pack.info(obj.pack_offset) - stream = pack.stream(obj.pack_offset) - - assert info.pack_offset == stream.pack_offset - assert info.type_id == stream.type_id - assert hasattr(stream, 'read') - - # it should be possible to read from both streams - assert obj.read() == stream.read() - - streams = pack.collect_streams(obj.pack_offset) - assert streams - - # read the stream - try: - dstream = DeltaApplyReader.new(streams) - except ValueError: - # ignore these, old git versions use only ref deltas, - # which we havent resolved ( as we are without an index ) - # Also ignore non-delta streams - continue - # END get deltastream - - # read all - data = dstream.read() - assert len(data) == dstream.size - - # test seek - dstream.seek(0) - assert dstream.read() == data - - - # read chunks - # NOTE: the current implementation is safe, it basically transfers - # all calls to the underlying memory map - - # END for each object - assert num_obj == size - - - def test_pack_index(self): - # check version 1 and 2 - for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): - index = PackIndexFile(indexfile) - self._assert_index_file(index, version, size) - # END run tests - - def test_pack(self): - # there is this special version 3, but apparently its like 2 ... - for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): - pack = PackFile(packfile) - self._assert_pack_file(pack, version, size) - # END for each pack to test - - @with_rw_directory - def test_pack_entity(self, rw_dir): - pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2), - (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): - packfile, version, size = packinfo - indexfile, version, size = indexinfo - entity = PackEntity(packfile) - assert entity.pack().path() == packfile - assert entity.index().path() == indexfile - pack_objs.extend(entity.stream_iter()) - - count = 0 - for info, stream in izip(entity.info_iter(), entity.stream_iter()): - count += 1 - assert info.binsha == stream.binsha - assert len(info.binsha) == 20 - assert info.type_id == stream.type_id - assert info.size == stream.size - - # we return fully resolved items, which is implied by the sha centric access - assert not info.type_id in delta_types - - # try all calls - assert len(entity.collect_streams(info.binsha)) - oinfo = entity.info(info.binsha) - assert isinstance(oinfo, OInfo) - assert oinfo.binsha is not None - ostream = entity.stream(info.binsha) - assert isinstance(ostream, OStream) - assert ostream.binsha is not None - - # verify the stream - try: - assert entity.is_valid_stream(info.binsha, use_crc=True) - except UnsupportedOperation: - pass - # END ignore version issues - assert entity.is_valid_stream(info.binsha, use_crc=False) - # END for each info, stream tuple - assert count == size - - # END for each entity - - # pack writing - write all packs into one - # index path can be None - pack_path = tempfile.mktemp('', "pack", rw_dir) - index_path = tempfile.mktemp('', 'index', rw_dir) - iteration = 0 - def rewind_streams(): - for obj in pack_objs: - obj.stream.seek(0) - #END utility - for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): - pfile = open(ppath, 'wb') - iwrite = None - if ipath: - ifile = open(ipath, 'wb') - iwrite = ifile.write - #END handle ip - - # make sure we rewind the streams ... we work on the same objects over and over again - if iteration > 0: - rewind_streams() - #END rewind streams - iteration += 1 - - pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) - pfile.close() - assert os.path.getsize(ppath) > 100 - - # verify pack - pf = PackFile(ppath) - assert pf.size() == len(pack_objs) - assert pf.version() == PackFile.pack_version_default - assert pf.checksum() == pack_sha - - # verify index - if ipath is not None: - ifile.close() - assert os.path.getsize(ipath) > 100 - idx = PackIndexFile(ipath) - assert idx.version() == PackIndexFile.index_version_default - assert idx.packfile_checksum() == pack_sha - assert idx.indexfile_checksum() == index_sha - assert idx.size() == len(pack_objs) - #END verify files exist - #END for each packpath, indexpath pair - - # verify the packs throughly - rewind_streams() - entity = PackEntity.create(pack_objs, rw_dir) - count = 0 - for info in entity.info_iter(): - count += 1 - for use_crc in range(2): - assert entity.is_valid_stream(info.binsha, use_crc) - # END for each crc mode - #END for each info - assert count == len(pack_objs) - - - def test_pack_64(self): - # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets - # of course without really needing such a huge pack - raise SkipTest() + + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) + + + def _assert_index_file(self, index, version, size): + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size + assert len(index.offsets()) == size + + # get all data of all objects + for oidx in xrange(index.size()): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4,8,11,17,20): + assert index.partial_sha_to_index(sha[:l], l*2) == oidx + + # END for each object index in indexfile + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) + + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = DeltaApplyReader.new(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we havent resolved ( as we are without an index ) + # Also ignore non-delta streams + continue + # END get deltastream + + # read all + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + + + # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map + + # END for each object + assert num_obj == size + + + def test_pack_index(self): + # check version 1 and 2 + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_objs = list() + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + pack_objs.extend(entity.stream_iter()) + + count = 0 + for info, stream in izip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # try all calls + assert len(entity.collect_streams(info.binsha)) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None + + # verify the stream + try: + assert entity.is_valid_stream(info.binsha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.binsha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path = tempfile.mktemp('', "pack", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + iteration = 0 + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + #END utility + for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + pfile = open(ppath, 'wb') + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write + #END handle ip + + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + rewind_streams() + #END rewind streams + iteration += 1 + + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pfile.close() + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == pack_sha + + # verify index + if ipath is not None: + ifile.close() + assert os.path.getsize(ipath) > 100 + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) + #END verify files exist + #END for each packpath, indexpath pair + + # verify the packs throughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in range(2): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + #END for each info + assert count == len(pack_objs) + + + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + raise SkipTest() diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 523f77056..6dc27463c 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -4,24 +4,24 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( - TestBase, - DummyStream, - Sha1Writer, - make_bytes, - make_object, - fixture_path - ) + TestBase, + DummyStream, + Sha1Writer, + make_bytes, + make_object, + fixture_path + ) from gitdb import * from gitdb.util import ( - NULL_HEX_SHA, - hex_to_bin - ) + NULL_HEX_SHA, + hex_to_bin + ) from gitdb.util import zlib from gitdb.typ import ( - str_blob_type - ) + str_blob_type + ) import time import tempfile @@ -31,124 +31,124 @@ class TestStream(TestBase): - """Test stream classes""" - - data_sizes = (15, 10000, 1000*1024+512) - - def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): - """Make stream tests - the orig_stream is seekable, allowing it to be - rewound and reused - :param cdata: the data we expect to read from stream, the contents - :param rewind_stream: function called to rewind the stream to make it ready - for reuse""" - ns = 10 - assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) - - # read in small steps - ss = len(cdata) / ns - for i in range(ns): - data = stream.read(ss) - chunk = cdata[i*ss:(i+1)*ss] - assert data == chunk - # END for each step - rest = stream.read() - if rest: - assert rest == cdata[-len(rest):] - # END handle rest - - if isinstance(stream, DecompressMemMapReader): - assert len(stream.data()) == stream.compressed_bytes_read() - # END handle special type - - rewind_stream(stream) - - # read everything - rdata = stream.read() - assert rdata == cdata - - if isinstance(stream, DecompressMemMapReader): - assert len(stream.data()) == stream.compressed_bytes_read() - # END handle special type - - def test_decompress_reader(self): - for close_on_deletion in range(2): - for with_size in range(2): - for ds in self.data_sizes: - cdata = make_bytes(ds, randomize=False) - - # zdata = zipped actual data - # cdata = original content data - - # create reader - if with_size: - # need object data - zdata = zlib.compress(make_object(str_blob_type, cdata)) - type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) - assert size == len(cdata) - assert type == str_blob_type - - # even if we don't set the size, it will be set automatically on first read - test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) - assert test_reader._s == len(cdata) - else: - # here we need content data - zdata = zlib.compress(cdata) - reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) - assert reader._s == len(cdata) - # END get reader - - self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) - - # put in a dummy stream for closing - dummy = DummyStream() - reader._m = dummy - - assert not dummy.closed - del(reader) - assert dummy.closed == close_on_deletion - # END for each datasize - # END whether size should be used - # END whether stream should be closed when deleted - - def test_sha_writer(self): - writer = Sha1Writer() - assert 2 == writer.write("hi") - assert len(writer.sha(as_hex=1)) == 40 - assert len(writer.sha(as_hex=0)) == 20 - - # make sure it does something ;) - prev_sha = writer.sha() - writer.write("hi again") - assert writer.sha() != prev_sha - - def test_compressed_writer(self): - for ds in self.data_sizes: - fd, path = tempfile.mkstemp() - ostream = FDCompressedSha1Writer(fd) - data = make_bytes(ds, randomize=False) - - # for now, just a single write, code doesn't care about chunking - assert len(data) == ostream.write(data) - ostream.close() - - # its closed already - self.failUnlessRaises(OSError, os.close, fd) - - # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) - written_data = os.read(fd, os.path.getsize(path)) - assert len(written_data) == os.path.getsize(path) - os.close(fd) - assert written_data == zlib.compress(data, 1) # best speed - - os.remove(path) - # END for each os - - def test_decompress_reader_special_case(self): - odb = LooseObjectDB(fixture_path('objects')) - ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) - - # if there is a bug, we will be missing one byte exactly ! - data = ostream.read() - assert len(data) == ostream.size - + """Test stream classes""" + + data_sizes = (15, 10000, 1000*1024+512) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) / ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i*ss:(i+1)*ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(str_blob_type, cdata)) + type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert type == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write("hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write("hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + + # its closed already + self.failUnlessRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 90f4156b9..35f9f44a7 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -8,98 +8,98 @@ from lib import TestBase from gitdb.util import ( - to_hex_sha, - to_bin_sha, - NULL_HEX_SHA, - LockedFD - ) + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, + LockedFD + ) - + class TestUtils(TestBase): - def test_basics(self): - assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA - assert len(to_bin_sha(NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA - - def _cmp_contents(self, file_path, data): - # raise if data from file at file_path - # does not match data string - fp = open(file_path, "rb") - try: - assert fp.read() == data - finally: - fp.close() - - def test_lockedfd(self): - my_file = tempfile.mktemp() - orig_data = "hello" - new_data = "world" - my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data) - my_file_fp.close() - - try: - lfd = LockedFD(my_file) - lockfilepath = lfd._lockfilepath() - - # cannot end before it was started - self.failUnlessRaises(AssertionError, lfd.rollback) - self.failUnlessRaises(AssertionError, lfd.commit) - - # open for writing - assert not os.path.isfile(lockfilepath) - wfd = lfd.open(write=True) - assert lfd._fd is wfd - assert os.path.isfile(lockfilepath) - - # write data and fail - os.write(wfd, new_data) - lfd.rollback() - assert lfd._fd is None - self._cmp_contents(my_file, orig_data) - assert not os.path.isfile(lockfilepath) - - # additional call doesnt fail - lfd.commit() - lfd.rollback() - - # test reading - lfd = LockedFD(my_file) - rfd = lfd.open(write=False) - assert os.read(rfd, len(orig_data)) == orig_data - - assert os.path.isfile(lockfilepath) - # deletion rolls back - del(lfd) - assert not os.path.isfile(lockfilepath) - - - # write data - concurrently - lfd = LockedFD(my_file) - olfd = LockedFD(my_file) - assert not os.path.isfile(lockfilepath) - wfdstream = lfd.open(write=True, stream=True) # this time as stream - assert os.path.isfile(lockfilepath) - # another one fails - self.failUnlessRaises(IOError, olfd.open) - - wfdstream.write(new_data) - lfd.commit() - assert not os.path.isfile(lockfilepath) - self._cmp_contents(my_file, new_data) - - # could test automatic _end_writing on destruction - finally: - os.remove(my_file) - # END final cleanup - - # try non-existing file for reading - lfd = LockedFD(tempfile.mktemp()) - try: - lfd.open(write=False) - except OSError: - assert not os.path.exists(lfd._lockfilepath()) - else: - self.fail("expected OSError") - # END handle exceptions + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + fp = open(file_path, "rb") + try: + assert fp.read() == data + finally: + fp.close() + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + my_file_fp = open(my_file, "wb") + my_file_fp.write(orig_data) + my_file_fp.close() + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.failUnlessRaises(AssertionError, lfd.rollback) + self.failUnlessRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesnt fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.failUnlessRaises(IOError, olfd.open) + + wfdstream.write(new_data) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/gitdb/util.py b/gitdb/util.py index 013f5fc78..1662b662d 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -13,28 +13,28 @@ # in py 2.4, StringIO is only StringI, without write support. # Hence we must use the python implementation for this if sys.version_info[1] < 5: - from StringIO import StringIO + from StringIO import StringIO # END handle python 2.4 try: - import async.mod.zlib as zlib + import async.mod.zlib as zlib except ImportError: - import zlib + import zlib # END try async zlib from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer + ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. if sys.version_info[1] < 6: - mman = StaticWindowMapManager() + mman = StaticWindowMapManager() else: - mman = SlidingWindowMapManager() + mman = SlidingWindowMapManager() #END handle mman try: @@ -43,19 +43,19 @@ import sha try: - from struct import unpack_from + from struct import unpack_from except ImportError: - from struct import unpack, calcsize - __calcsize_cache = dict() - def unpack_from(fmt, data, offset=0): - try: - size = __calcsize_cache[fmt] - except KeyError: - size = calcsize(fmt) - __calcsize_cache[fmt] = size - # END exception handling - return unpack(fmt, data[offset : offset + size]) - # END own unpack_from implementation + from struct import unpack, calcsize + __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset : offset + size]) + # END own unpack_from implementation #{ Globals @@ -100,25 +100,25 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... class _RandomAccessStringIO(object): - """Wrapper to provide required functionality in case memory maps cannot or may - not be used. This is only really required in python 2.4""" - __slots__ = '_sio' - - def __init__(self, buf=''): - self._sio = StringIO(buf) - - def __getattr__(self, attr): - return getattr(self._sio, attr) - - def __len__(self): - return len(self.getvalue()) - - def __getitem__(self, i): - return self.getvalue()[i] - - def __getslice__(self, start, end): - return self.getvalue()[start:end] - + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = StringIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + #} END compatibility stuff ... #{ Routines @@ -134,85 +134,85 @@ def make_sha(source=''): return sha1 def allocate_memory(size): - """:return: a file-protocol accessible memory block of the given size""" - if size == 0: - return _RandomAccessStringIO('') - # END handle empty chunks gracefully - - try: - return mmap.mmap(-1, size) # read-write by default - except EnvironmentError: - # setup real memory instead - # this of course may fail if the amount of memory is not available in - # one chunk - would only be the case in python 2.4, being more likely on - # 32 bit systems. - return _RandomAccessStringIO("\0"*size) - # END handle memory allocation - + """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessStringIO('') + # END handle empty chunks gracefully + + try: + return mmap.mmap(-1, size) # read-write by default + except EnvironmentError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return _RandomAccessStringIO("\0"*size) + # END handle memory allocation + def file_contents_ro(fd, stream=False, allow_mmap=True): - """:return: read-only contents of the file represented by the file descriptor fd - - :param fd: file descriptor opened for reading - :param stream: if False, random access is provided, otherwise the stream interface - is provided. - :param allow_mmap: if True, its allowed to map the contents into memory, which - allows large files to be handled and accessed efficiently. The file-descriptor - will change its position if this is False""" - try: - if allow_mmap: - # supports stream and random access - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - except EnvironmentError: - # python 2.4 issue, 0 wants to be the actual size - return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) - # END handle python 2.4 - except OSError: - pass - # END exception handling - - # read manully - contents = os.read(fd, os.fstat(fd).st_size) - if stream: - return _RandomAccessStringIO(contents) - return contents - + """:return: read-only contents of the file represented by the file descriptor fd + + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except EnvironmentError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 + except OSError: + pass + # END exception handling + + # read manully + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return _RandomAccessStringIO(contents) + return contents + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): - """Get the file contents at filepath as fast as possible - - :return: random access compatible memory of the given filepath - :param stream: see ``file_contents_ro`` - :param allow_mmap: see ``file_contents_ro`` - :param flags: additional flags to pass to os.open - :raise OSError: If the file could not be opened - - **Note** for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object - databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) - try: - return file_contents_ro(fd, stream, allow_mmap) - finally: - close(fd) - # END assure file is closed - + """Get the file contents at filepath as fast as possible + + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + def sliding_ro_buffer(filepath, flags=0): - """ - :return: a buffer compatible object which uses our mapped memory manager internally - ready to read the whole given filepath""" - return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) - + """ + :return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): - """:return: hexified version of sha""" - if len(sha) == 40: - return sha - return bin_to_hex(sha) - + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + def to_bin_sha(sha): - if len(sha) == 20: - return sha - return hex_to_bin(sha) + if len(sha) == 20: + return sha + return hex_to_bin(sha) #} END routines @@ -221,162 +221,162 @@ def to_bin_sha(sha): #{ Utilities class LazyMixin(object): - """ - Base class providing an interface to lazily retrieve attribute values upon - first access. If slots are used, memory will only be reserved once the attribute - is actually accessed and retrieved the first time. All future accesses will - return the cached value as stored in the Instance's dict or slot. - """ - - __slots__ = tuple() - - def __getattr__(self, attr): - """ - Whenever an attribute is requested that we do not know, we allow it - to be created and set. Next time the same attribute is reqeusted, it is simply - returned from our dict/slots. """ - self._set_cache_(attr) - # will raise in case the cache was not created - return object.__getattribute__(self, attr) - - def _set_cache_(self, attr): - """ - This method should be overridden in the derived class. - It should check whether the attribute named by attr can be created - and cached. Do nothing if you do not know the attribute or call your subclass - - The derived class may create as many additional attributes as it deems - necessary in case a git command returns more information than represented - in the single attribute.""" - pass - - + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is reqeusted, it is simply + returned from our dict/slots. """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ + This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + + class LockedFD(object): - """ - This class facilitates a safe read and write operation to a file on disk. - If we write to 'file', we obtain a lock file at 'file.lock' and write to - that instead. If we succeed, the lock file will be renamed to overwrite - the original file. - - When reading, we obtain a lock file, but to prevent other writers from - succeeding while we are reading the file. - - This type handles error correctly in that it will assure a consistent state - on destruction. - - **note** with this setup, parallel reading is not possible""" - __slots__ = ("_filepath", '_fd', '_write') - - def __init__(self, filepath): - """Initialize an instance with the givne filepath""" - self._filepath = filepath - self._fd = None - self._write = None # if True, we write a file - - def __del__(self): - # will do nothing if the file descriptor is already closed - if self._fd is not None: - self.rollback() - - def _lockfilepath(self): - return "%s.lock" % self._filepath - - def open(self, write=False, stream=False): - """ - Open the file descriptor for reading or writing, both in binary mode. - - :param write: if True, the file descriptor will be opened for writing. Other - wise it will be opened read-only. - :param stream: if True, the file descriptor will be wrapped into a simple stream - object which supports only reading or writing - :return: fd to read from or write to. It is still maintained by this instance - and must not be closed directly - :raise IOError: if the lock could not be retrieved - :raise OSError: If the actual file could not be opened for reading - - **note** must only be called once""" - if self._write is not None: - raise AssertionError("Called %s multiple times" % self.open) - - self._write = write - - # try to open the lock file - binary = getattr(os, 'O_BINARY', 0) - lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary - try: - fd = os.open(self._lockfilepath(), lockmode, 0600) - if not write: - os.close(fd) - else: - self._fd = fd - # END handle file descriptor - except OSError: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) - # END handle lock retrieval - - # open actual file if required - if self._fd is None: - # we could specify exlusive here, as we obtained the lock anyway - try: - self._fd = os.open(self._filepath, os.O_RDONLY | binary) - except: - # assure we release our lockfile - os.remove(self._lockfilepath()) - raise - # END handle lockfile - # END open descriptor for reading - - if stream: - # need delayed import - from stream import FDStream - return FDStream(self._fd) - else: - return self._fd - # END handle stream - - def commit(self): - """When done writing, call this function to commit your changes into the - actual file. - The file descriptor will be closed, and the lockfile handled. - - **Note** can be called multiple times""" - self._end_writing(successful=True) - - def rollback(self): - """Abort your operation without any changes. The file descriptor will be - closed, and the lock released. - - **Note** can be called multiple times""" - self._end_writing(successful=False) - - def _end_writing(self, successful=True): - """Handle the lock according to the write mode """ - if self._write is None: - raise AssertionError("Cannot end operation if it wasn't started yet") - - if self._fd is None: - return - - os.close(self._fd) - self._fd = None - - lockfile = self._lockfilepath() - if self._write and successful: - # on windows, rename does not silently overwrite the existing one - if sys.platform == "win32": - if isfile(self._filepath): - os.remove(self._filepath) - # END remove if exists - # END win32 special handling - os.rename(lockfile, self._filepath) - - # assure others can at least read the file - the tmpfile left it at rw-- - # We may also write that file, on windows that boils down to a remove- - # protection as well - chmod(self._filepath, 0644) - else: - # just delete the file so far, we failed - os.remove(lockfile) - # END successful handling + """ + This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + **note** with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """ + Open the file descriptor for reading or writing, both in binary mode. + + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + + **note** must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode, 0600) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exlusive here, as we obtained the lock anyway + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + os.remove(self._lockfilepath()) + raise + # END handle lockfile + # END open descriptor for reading + + if stream: + # need delayed import + from stream import FDStream + return FDStream(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + + **Note** can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + + **Note** can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if isfile(self._filepath): + os.remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, 0644) + else: + # just delete the file so far, we failed + os.remove(lockfile) + # END successful handling #} END utilities From 56d7a9b64b6d768dd118a02c1ed2afb38265c8b9 Mon Sep 17 00:00:00 2001 From: Yuriy Arhipov Date: Mon, 24 Feb 2014 02:08:58 +0400 Subject: [PATCH 0218/3719] [#7021] ticket:533 fixed error with pgp signed commits --- git/objects/commit.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 4ccd9d755..34ae15bf7 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -58,12 +58,12 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): __slots__ = ("tree", "author", "authored_date", "author_tz_offset", "committer", "committed_date", "committer_tz_offset", - "message", "parents", "encoding") + "message", "parents", "encoding", "gpgsig") _id_attribute_ = "binsha" def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, author_tz_offset=None, committer=None, committed_date=None, committer_tz_offset=None, - message=None, parents=None, encoding=None): + message=None, parents=None, encoding=None, gpgsig=None): """Instantiate a new Commit. All keyword arguments taking None as default will be implicitly set on first query. @@ -121,6 +121,8 @@ def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, aut self.parents = parents if encoding is not None: self.encoding = encoding + if gpgsig is not None: + self.gpgsig = gpgsig @classmethod def _get_intermediate_items(cls, commit): @@ -439,15 +441,29 @@ def _deserialize(self, stream): # now we can have the encoding line, or an empty line followed by the optional # message. self.encoding = self.default_encoding - # read encoding or empty line to separate message + + # read headers enc = next_line - enc = enc.strip() - if enc: - self.encoding = enc[enc.find(' ')+1:] - # now comes the message separator - readline() - # END handle encoding - + buf = enc.strip() + while buf != "": + if buf[0:10] == "encoding ": + self.encoding = buf[buf.find(' ')+1:] + elif buf[0:7] == "gpgsig ": + sig = buf[buf.find(' ')+1:] + "\n" + is_next_header = False + while True: + sigbuf = readline() + if sigbuf == "": break + if sigbuf[0:1] != " ": + buf = sigbuf.strip() + is_next_header = True + break + sig += sigbuf[1:] + self.gpgsig = sig.rstrip("\n") + if is_next_header: + continue + buf = readline().strip() + # decode the authors name try: self.author.name = self.author.name.decode(self.encoding) From 8005591231c8ae329f0ff320385b190d2ea81df0 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Mon, 3 Mar 2014 23:09:39 +0000 Subject: [PATCH 0219/3719] [#7021] Added serialization and test from upstream and fixed test issues --- git/objects/commit.py | 5 ++++ git/test/fixtures/commit_with_gpgsig | 30 ++++++++++++++++++++ git/test/lib/helper.py | 2 +- git/test/test_commit.py | 42 ++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 git/test/fixtures/commit_with_gpgsig diff --git a/git/objects/commit.py b/git/objects/commit.py index 34ae15bf7..035ce004e 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -398,6 +398,11 @@ def _serialize(self, stream): if self.encoding != self.default_encoding: write("encoding %s\n" % self.encoding) + + if self.gpgsig: + write("gpgsig") + for sigline in self.gpgsig.rstrip("\n").split("\n"): + write(" "+sigline+"\n") write("\n") diff --git a/git/test/fixtures/commit_with_gpgsig b/git/test/fixtures/commit_with_gpgsig new file mode 100644 index 000000000..f38cdabd6 --- /dev/null +++ b/git/test/fixtures/commit_with_gpgsig @@ -0,0 +1,30 @@ +tree cefbccb4843d821183ae195e70a17c9938318945 +parent 904435cf76a9bdd5eb41b1c4e049d5a64f3a8400 +author Jon Mason 1367013117 -0700 +committer Jon Mason 1368640702 -0700 +gpgsig -----BEGIN PGP SIGNATURE----- + Version: GnuPG v1.4.11 (GNU/Linux) + + iQIcBAABAgAGBQJRk8zMAAoJEG5mS6x6i9IjsTEP/0v2Wx/i7dqyKban6XMIhVdj + uI0DycfXqnCCZmejidzeao+P+cuK/ZAA/b9fU4MtwkDm2USvnIOrB00W0isxsrED + sdv6uJNa2ybGjxBolLrfQcWutxGXLZ1FGRhEvkPTLMHHvVriKoNFXcS7ewxP9MBf + NH97K2wauqA+J4BDLDHQJgADCOmLrGTAU+G1eAXHIschDqa6PZMH5nInetYZONDh + 3SkOOv8VKFIF7gu8X7HC+7+Y8k8U0TW0cjlQ2icinwCc+KFoG6GwXS7u/VqIo1Yp + Tack6sxIdK7NXJhV5gAeAOMJBGhO0fHl8UUr96vGEKwtxyZhWf8cuIPOWLk06jA0 + g9DpLqmy/pvyRfiPci+24YdYRBua/vta+yo/Lp85N7Hu/cpIh+q5WSLvUlv09Dmo + TTTG8Hf6s3lEej7W8z2xcNZoB6GwXd8buSDU8cu0I6mEO9sNtAuUOHp2dBvTA6cX + PuQW8jg3zofnx7CyNcd3KF3nh2z8mBcDLgh0Q84srZJCPRuxRcp9ylggvAG7iaNd + XMNvSK8IZtWLkx7k3A3QYt1cN4y1zdSHLR2S+BVCEJea1mvUE+jK5wiB9S4XNtKm + BX/otlTa8pNE3fWYBxURvfHnMY4i3HQT7Bc1QjImAhMnyo2vJk4ORBJIZ1FTNIhJ + JzJMZDRLQLFvnzqZuCjE + =przd + -----END PGP SIGNATURE----- + +NTB: Multiple NTB client fix + +Fix issue with adding multiple ntb client devices to the ntb virtual +bus. Previously, multiple devices would be added with the same name, +resulting in crashes. To get around this issue, add a unique number to +the device when it is added. + +Signed-off-by: Jon Mason diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 3a60d116c..5790a8589 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -227,7 +227,7 @@ class TestBase(TestCase): """ @classmethod - def setUpAll(cls): + def setUpClass(cls): """ Dynamically add a read-only repository to our actual type. This way each test type has its own repository diff --git a/git/test/test_commit.py b/git/test/test_commit.py index 4a8d8b878..0b7ed9ffb 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -13,6 +13,7 @@ from cStringIO import StringIO import time import sys +import re def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False): @@ -273,3 +274,44 @@ def test_serialization_unicode_support(self): # it appears cmt.author.__repr__() + def test_gpgsig(self): + cmt = self.rorepo.commit() + cmt._deserialize(open(fixture_path('commit_with_gpgsig'))) + + fixture_sig = """-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.11 (GNU/Linux) + +iQIcBAABAgAGBQJRk8zMAAoJEG5mS6x6i9IjsTEP/0v2Wx/i7dqyKban6XMIhVdj +uI0DycfXqnCCZmejidzeao+P+cuK/ZAA/b9fU4MtwkDm2USvnIOrB00W0isxsrED +sdv6uJNa2ybGjxBolLrfQcWutxGXLZ1FGRhEvkPTLMHHvVriKoNFXcS7ewxP9MBf +NH97K2wauqA+J4BDLDHQJgADCOmLrGTAU+G1eAXHIschDqa6PZMH5nInetYZONDh +3SkOOv8VKFIF7gu8X7HC+7+Y8k8U0TW0cjlQ2icinwCc+KFoG6GwXS7u/VqIo1Yp +Tack6sxIdK7NXJhV5gAeAOMJBGhO0fHl8UUr96vGEKwtxyZhWf8cuIPOWLk06jA0 +g9DpLqmy/pvyRfiPci+24YdYRBua/vta+yo/Lp85N7Hu/cpIh+q5WSLvUlv09Dmo +TTTG8Hf6s3lEej7W8z2xcNZoB6GwXd8buSDU8cu0I6mEO9sNtAuUOHp2dBvTA6cX +PuQW8jg3zofnx7CyNcd3KF3nh2z8mBcDLgh0Q84srZJCPRuxRcp9ylggvAG7iaNd +XMNvSK8IZtWLkx7k3A3QYt1cN4y1zdSHLR2S+BVCEJea1mvUE+jK5wiB9S4XNtKm +BX/otlTa8pNE3fWYBxURvfHnMY4i3HQT7Bc1QjImAhMnyo2vJk4ORBJIZ1FTNIhJ +JzJMZDRLQLFvnzqZuCjE +=przd +-----END PGP SIGNATURE-----""" + self.assertEqual(cmt.gpgsig, fixture_sig) + self.assertIn('NTB: Multiple NTB client fix', cmt.message) + cmt.gpgsig = "" + self.assertNotEqual(cmt.gpgsig, fixture_sig) + + cstream = StringIO() + cmt._serialize(cstream) + value = cstream.getvalue() + self.assertRegexpMatches(value, re.compile(r"^gpgsig $", re.MULTILINE)) + + cstream.seek(0) + cmt.gpgsig = None + cmt._deserialize(cstream) + self.assertEqual(cmt.gpgsig, "") + + cmt.gpgsig = None + cstream = StringIO() + cmt._serialize(cstream) + value = cstream.getvalue() + self.assertNotRegexpMatches(value, re.compile(r"^gpgsig ", re.MULTILINE)) From f7ed51ba4c8416888f5744ddb84726316c461051 Mon Sep 17 00:00:00 2001 From: Cory Johns Date: Tue, 4 Mar 2014 18:33:13 +0000 Subject: [PATCH 0220/3719] [#7021] Fixed error serializing programmatically created commits --- git/objects/commit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 035ce004e..edbdf038c 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -121,7 +121,7 @@ def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, aut self.parents = parents if encoding is not None: self.encoding = encoding - if gpgsig is not None: + if binsha == '\x00'*20 or gpgsig is not None: self.gpgsig = gpgsig @classmethod From 9d0473c1d1e6cadd986102712fff9196fff96212 Mon Sep 17 00:00:00 2001 From: firm1 Date: Mon, 24 Mar 2014 14:49:33 +0100 Subject: [PATCH 0221/3719] update commit function --- git/index/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 3bd8634c7..0c1b68d94 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -873,7 +873,7 @@ def move(self, items, skip_errors=False, **kwargs): return out - def commit(self, message, parent_commits=None, head=True): + def def commit(self, message, parent_commits=None, head=True, author=None, committer=None): """Commit the current default index file, creating a commit object. For more information on the arguments, see tree.commit. @@ -884,7 +884,7 @@ def commit(self, message, parent_commits=None, head=True): :return: Commit object representing the new commit""" tree = self.write_tree() - return Commit.create_from_tree(self.repo, tree, message, parent_commits, head) + return Commit.create_from_tree(self.repo, tree, message, parent_commits, head, author=author, committer=committer) @classmethod def _flush_stdin_and_wait(cls, proc, ignore_stdout = False): From 5d602f267c32e1e917599d9bcdcfec4eef05d477 Mon Sep 17 00:00:00 2001 From: firm1 Date: Mon, 24 Mar 2014 14:52:44 +0100 Subject: [PATCH 0222/3719] add param to create_from_tree --- git/objects/commit.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index cbfd5097b..f1c2a23d1 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -254,7 +254,7 @@ def _iter_from_process_or_stream(cls, repo, proc_or_stream): @classmethod - def create_from_tree(cls, repo, tree, message, parent_commits=None, head=False): + def create_from_tree(cls, repo, tree, message, parent_commits=None, head=False, author=None, committer=None): """Commit the given tree, creating a commit object. :param repo: Repo object the commit should be part of @@ -299,8 +299,13 @@ def create_from_tree(cls, repo, tree, message, parent_commits=None, head=False): cr = repo.config_reader() env = os.environ - committer = Actor.committer(cr) - author = Actor.author(cr) + if author is None and committer is None: + committer = Actor.committer(cr) + author = Actor.author(cr) + elif author is None: + author = Actor.author(cr) + elif committer is None: + committer = Actor.committer(cr) # PARSE THE DATES unix_time = int(time()) From 28fdf05b1d7827744b7b70eeb1cc66d3afd38c82 Mon Sep 17 00:00:00 2001 From: firm1 Date: Mon, 24 Mar 2014 14:54:23 +0100 Subject: [PATCH 0223/3719] correct log reference --- git/refs/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/log.py b/git/refs/log.py index 9a719ec06..560ffd3e1 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -247,7 +247,7 @@ def append_entry(cls, config_reader, filepath, oldbinsha, newbinsha, message): raise ValueError("Shas need to be given in binary format") #END handle sha type assure_directory_exists(filepath, is_file=True) - entry = RefLogEntry((bin_to_hex(oldbinsha), bin_to_hex(newbinsha), Actor.committer(config_reader), (int(time.time()), time.altzone), message)) + entry = RefLogEntry((bin_to_hex(oldbinsha), bin_to_hex(newbinsha), config_reader, (int(time.time()), time.altzone), message)) lf = LockFile(filepath) lf._obtain_lock_or_raise() From 4a7e7a769087b1790a18d6645740b5b670f5086b Mon Sep 17 00:00:00 2001 From: firm1 Date: Mon, 24 Mar 2014 14:56:02 +0100 Subject: [PATCH 0224/3719] Update symbolic.py --- git/refs/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index ef21950fa..5374285e0 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -355,7 +355,7 @@ def log_append(self, oldbinsha, message, newbinsha=None): :param newbinsha: The sha the ref points to now. If None, our current commit sha will be used :return: added RefLogEntry instance""" - return RefLog.append_entry(self.repo.config_reader(), RefLog.path(self), oldbinsha, + return RefLog.append_entry(self.commit.committer, RefLog.path(self), oldbinsha, (newbinsha is None and self.commit.binsha) or newbinsha, message) From 56cc93a548f35a0becd49a7eacde86f55ffc5dc5 Mon Sep 17 00:00:00 2001 From: Tatsuki Sugiura Date: Tue, 8 May 2012 09:18:36 +0900 Subject: [PATCH 0225/3719] Fix fd leak on git cmd. Currently if command is called with as_proces=True, pipes for the command will not be closed. This change makes sure to close command file descriptors. Conflicts: git/cmd.py --- git/cmd.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/git/cmd.py b/git/cmd.py index 2d4aa7279..c342148fc 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -73,6 +73,9 @@ def __init__(self, proc, args ): self.args = args def __del__(self): + self.proc.stdout.close() + self.proc.stderr.close() + # did the process finish already so we have a return code ? if self.proc.poll() is not None: return @@ -100,6 +103,8 @@ def wait(self): :raise GitCommandError: if the return status is not 0""" status = self.proc.wait() + self.proc.stdout.close() + self.proc.stderr.close() if status != 0: raise GitCommandError(self.args, status, self.proc.stderr.read()) # END status handling From b137f55232155b16aa308ec4ea8d6bc994268b0d Mon Sep 17 00:00:00 2001 From: Tatsuki Sugiura Date: Tue, 8 May 2012 09:35:33 +0900 Subject: [PATCH 0226/3719] Ignore signal exception on AutoInterrupt destructor. When command run as subprocess, AutoInterrupt will kill the process on destructor. However, if process already finished, it raise OSError exception. This fix just ignore OSError on os.kill. Conflicts: git/cmd.py --- git/cmd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/cmd.py b/git/cmd.py index c342148fc..b8b27d42f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -87,6 +87,8 @@ def __del__(self): # try to kill it try: os.kill(self.proc.pid, 2) # interrupt signal + except OSError: + pass # ignore error when process already died except AttributeError: # try windows # for some reason, providing None for stdout/stderr still prints something. This is why From 3a1e0d7117b9e4ea4be3ef4895e8b2b4937ff98a Mon Sep 17 00:00:00 2001 From: firm1 Date: Wed, 9 Apr 2014 16:03:21 +0200 Subject: [PATCH 0227/3719] fix syntax error --- git/index/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/base.py b/git/index/base.py index 0c1b68d94..160d21bf3 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -873,7 +873,7 @@ def move(self, items, skip_errors=False, **kwargs): return out - def def commit(self, message, parent_commits=None, head=True, author=None, committer=None): + def commit(self, message, parent_commits=None, head=True, author=None, committer=None): """Commit the current default index file, creating a commit object. For more information on the arguments, see tree.commit. From ec0b85e2d4907fb5fcfc5724e0e8df59e752c0d1 Mon Sep 17 00:00:00 2001 From: Remi Rampin Date: Thu, 24 Apr 2014 14:03:25 -0400 Subject: [PATCH 0228/3719] Fixes creating a Repo for a submodule Fixes #155. --- git/repo/base.py | 7 ++++--- git/repo/fun.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 3bbcdb592..9ac471a6b 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -32,6 +32,7 @@ from fun import ( rev_parse, is_git_dir, + find_git_dir, touch ) @@ -108,8 +109,8 @@ def __init__(self, path=None, odbt = DefaultDBType): self.git_dir = curpath self._working_tree_dir = os.path.dirname(curpath) break - gitpath = join(curpath, '.git') - if is_git_dir(gitpath): + gitpath = find_git_dir(join(curpath, '.git')) + if gitpath is not None: self.git_dir = gitpath self._working_tree_dir = curpath break @@ -119,7 +120,7 @@ def __init__(self, path=None, odbt = DefaultDBType): # END while curpath if self.git_dir is None: - raise InvalidGitRepositoryError(epath) + raise InvalidGitRepositoryError(epath) self._bare = False try: diff --git a/git/repo/fun.py b/git/repo/fun.py index 7a8657ab5..2c49d8367 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -7,6 +7,7 @@ join, isdir, isfile, + dirname, hex_to_bin, bin_to_hex ) @@ -31,6 +32,18 @@ def is_git_dir(d): return False +def find_git_dir(d): + if is_git_dir(d): + return d + elif isfile(d): + with open(d) as fp: + content = fp.read().rstrip() + if content.startswith('gitdir: '): + d = join(dirname(d), content[8:]) + return find_git_dir(d) + return None + + def short_to_long(odb, hexsha): """:return: long hexadecimal sha1 from the given less-than-40 byte hexsha or None if no candidate could be found. From 9e91a05aabb73cca7acec1cc0e07d8a562e31b45 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 May 2014 11:20:43 +0200 Subject: [PATCH 0229/3719] Make sure that branches looking like a numeric scalar will not become number type in python. The latter will break code that assumes it will get a string. --- git/objects/submodule/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index f7dc1597f..99d54076a 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -895,7 +895,7 @@ def iter_items(cls, repo, parent_commit='HEAD'): u = parser.get_value(sms, 'url') b = cls.k_head_default if parser.has_option(sms, cls.k_head_option): - b = parser.get_value(sms, cls.k_head_option) + b = str(parser.get_value(sms, cls.k_head_option)) # END handle optional information # get the binsha From 6099ac8e04a8f734555efe4e78ad766d9a6fb70a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:23:09 +0200 Subject: [PATCH 0230/3719] Added travis CI support --- .travis.yml | 8 ++++++++ README.rst | 8 +++++--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..e7d5214c1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.5" + - "2.6" + - "2.7" + - "pypy" + +script: nosetests diff --git a/README.rst b/README.rst index 30bff0ded..84feb373c 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,8 @@ Although memory maps have many advantages, they represent a very limited system Overview ######## +.. image:: https://travis-ci.org/Byron/smmap.svg?branch=master :target: https://travis-ci.org/Byron/smmap + Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. @@ -58,17 +60,17 @@ The project is home on github at `https://github.com/Byron/smmap Date: Sun, 4 May 2014 16:29:43 +0200 Subject: [PATCH 0231/3719] pypy deactivated, it doesn't yet work --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e7d5214c1..563797656 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: python python: + - "2.4" - "2.5" - "2.6" - "2.7" - - "pypy" + # - "pypy" - no getrefcount script: nosetests From 616e9ceaf917e4d8f3cf2c145401b8069ce307dd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:33:55 +0200 Subject: [PATCH 0232/3719] Oh, travis doesn't support older python versions, fair enough --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 563797656..fdba549d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python python: - - "2.4" - - "2.5" - "2.6" - "2.7" # - "pypy" - no getrefcount From a4eddfe72b442666ba3acdf89929e9b65de45f45 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:38:55 +0200 Subject: [PATCH 0233/3719] Added initial configuration of gitdb --- README.rst | 3 +++ gitdb/.travis.yml | 7 +++++++ gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 gitdb/.travis.yml diff --git a/README.rst b/README.rst index 753eb701c..68f71ff61 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,9 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= + +.. image:: https://travis-ci.org/Byron/gitdb.svg?branch=master :target: https://travis-ci.org/Byron/gitdb + https://github.com/gitpython-developers/gitdb/issues LICENSE diff --git a/gitdb/.travis.yml b/gitdb/.travis.yml new file mode 100644 index 000000000..5c8791d90 --- /dev/null +++ b/gitdb/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.6" + - "2.7" + # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) + +script: nosetests diff --git a/gitdb/ext/async b/gitdb/ext/async index 571412931..90326fb86 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 571412931829200aff06a44b9c5524e122e524e9 +Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 1b3ab5598..616e9ceaf 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 1b3ab5598e93369282502d049d64cb2ca12839cb +Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd From e2c94bf6983b378f99db175cd7810986cf338c32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:43:26 +0200 Subject: [PATCH 0234/3719] argh, one level too low, didn't see it in sublime --- gitdb/.travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gitdb/.travis.yml => .travis.yml (100%) diff --git a/gitdb/.travis.yml b/.travis.yml similarity index 100% rename from gitdb/.travis.yml rename to .travis.yml From 39de1127459b73b862f2b779bb4565ad6b4bd625 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:44:48 +0200 Subject: [PATCH 0235/3719] Fixed travis build status url --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 68f71ff61..0fc0534e6 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -.. image:: https://travis-ci.org/Byron/gitdb.svg?branch=master :target: https://travis-ci.org/Byron/gitdb +.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb https://github.com/gitpython-developers/gitdb/issues From 1f225d4b8c3d7eb90038c246a289a18c7b655da2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:53:36 +0200 Subject: [PATCH 0236/3719] Added support for travis ci --- .travis.yml | 7 +++++++ README.rst => README.md | 38 ++++++++++++++++++-------------------- git/ext/gitdb | 2 +- 3 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 .travis.yml rename README.rst => README.md (81%) diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..6d91c8b6b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.6" + - "2.7" + # - "pypy" - won't work as smmap doesn't work (see gitdb/.travis.yml for details) + +script: nosetests diff --git a/README.rst b/README.md similarity index 81% rename from README.rst rename to README.md index 128e74c02..c081f25f0 100644 --- a/README.rst +++ b/README.md @@ -1,6 +1,6 @@ -========== -GitPython -========== +## GitPython + +.. image:: https://travis-ci.org/gitpython-developers/GitPython.svg?branch=master :target: https://travis-ci.org/gitpython-developers/GitPython GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. @@ -8,21 +8,20 @@ It provides abstractions of git objects for easy access of repository data, and The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. -REQUIREMENTS -============ +### REQUIREMENTS * Git ( tested with 1.8.3.4 ) * Python Nose - used for running the tests - * Tested with nose 1.3.0 + - Tested with nose 1.3.0 * Mock by Michael Foord used for tests - * Tested with 1.0.1 + - Tested with 1.0.1 + +### INSTALL -INSTALL -======= If you have downloaded the source code: python setup.py install - + or if you want to obtain a copy more easily: pip install gitpython @@ -31,8 +30,8 @@ A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython -SOURCE -====== +### SOURCE + GitPython's git repo is available on GitHub, which can be browsed at: @@ -43,23 +42,22 @@ and cloned using: git clone git://github.com/gitpython-developers/GitPython.git git-python -DOCUMENTATION -============= +### DOCUMENTATION + The html-compiled documentation can be found at the following URL: http://packages.python.org/GitPython/ -MAILING LIST -============ +### MAILING LIST + http://groups.google.com/group/git-python -ISSUE TRACKER -============= +### ISSUE TRACKER + Issues are tracked on github: https://github.com/gitpython-developers/GitPython/issues -LICENSE -======= +### LICENSE New BSD License. See the LICENSE file. diff --git a/git/ext/gitdb b/git/ext/gitdb index 6576d5503..39de11274 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 6576d5503a64d124fd7bcf639cc8955918b3ac43 +Subproject commit 39de1127459b73b862f2b779bb4565ad6b4bd625 From cc93c4f3ddade455cc4f55bc93167b1d2aeddc4f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:57:18 +0200 Subject: [PATCH 0237/3719] Fixed travis-ci url in Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c081f25f0..6f2039dc1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## GitPython -.. image:: https://travis-ci.org/gitpython-developers/GitPython.svg?branch=master :target: https://travis-ci.org/gitpython-developers/GitPython +[![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. From 3b9b6fe0dd99803c80a3a3c52f003614ad3e0adf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 17:01:59 +0200 Subject: [PATCH 0238/3719] Let's see if recursive checkouts will fix 'gitdb not found' issue for travis. If not, pip install should do the job --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6d91c8b6b..48c05b7aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,4 +4,7 @@ python: - "2.7" # - "pypy" - won't work as smmap doesn't work (see gitdb/.travis.yml for details) -script: nosetests +install: + - git submodule update --init --recursive +script: + - nosetests From d6192ad1aed30adc023621089fdf845aa528dde9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 17:05:38 +0200 Subject: [PATCH 0239/3719] tags seem to be required for the tests to run - git-python usess objects from its own repo, tags are known to be stable --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 48c05b7aa..2a56beccf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,6 @@ python: install: - git submodule update --init --recursive + - git fetch --tags script: - nosetests From cf1cf469f315df00ce7b9d47693008cd2fa1b56a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 May 2014 09:31:15 +0200 Subject: [PATCH 0240/3719] Get starten on py3.3 compatability This is likely to fail, but lets see. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fdba549d4..d8640a08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.3" # - "pypy" - no getrefcount script: nosetests From 32d34eba0412770c382a1428a0ebb221dbba4187 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 May 2014 09:55:32 +0200 Subject: [PATCH 0241/3719] Test with python 3.3 as well Have to start making them compatible at some point. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5c8791d90..ff263f20d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.3" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) script: nosetests From 4a023acbe9fc9a183c395be969b7fc7d472490cb Mon Sep 17 00:00:00 2001 From: Maximiliano Curia Date: Tue, 6 May 2014 12:18:13 +0200 Subject: [PATCH 0242/3719] Fix for untracked_files no longer detected #138 --- git/repo/base.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 9ac471a6b..977801052 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -513,35 +513,33 @@ def is_dirty(self, index=True, working_tree=True, untracked_files=False): return True # END untracked files return False - + @property def untracked_files(self): """ :return: list(str,...) - - Files currently untracked as they have not been staged yet. Paths + + Files currently untracked as they have not been staged yet. Paths are relative to the current working directory of the git command. - + :note: ignored files will not appear here, i.e. files mentioned in .gitignore""" # make sure we get all files, no only untracked directores - proc = self.git.status(untracked_files=True, as_process=True) - stream = iter(proc.stdout) + proc = self.git.status(porcelain=True, + untracked_files=True, + as_process=True) + # Untracked files preffix in porcelain mode + prefix = "?? " untracked_files = list() - for line in stream: - if not line.startswith("# Untracked files:"): + for line in proc.stdout: + if not line.startswith(prefix): continue - # skip two lines - stream.next() - stream.next() - - for untracked_info in stream: - if not untracked_info.startswith("#\t"): - break - untracked_files.append(untracked_info.replace("#\t", "").rstrip()) - # END for each utracked info line - # END for each line + filename = line[len(preffix):].rstrip('\n') + # Special characters are escaped + if filename[0] == filename[-1] == '"': + filename = filename[1:-1].decode('string_escape') + untracked_files.append(filename) return untracked_files @property From 2ffde74dd7b5cbc4c018f0d608049be8eccc5101 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 May 2014 09:33:06 +0200 Subject: [PATCH 0243/3719] Updated readme with development status [skip ci] --- README.md | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6f2039dc1..d978917bc 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ ## GitPython -[![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) - GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. It provides abstractions of git objects for easy access of repository data, and additionally allows you to access the git repository more directly using either a pure python implementation, or the faster, but more resource intensive git command implementation. @@ -30,6 +28,41 @@ A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython +### DEVELOPMENT STATUS + +[![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) + +The project was idle for 2 years, the last release was made about 3 years ago. Reason for this might have been the project's dependency on me as sole active maintainer, which is an issue in itself. + +Now I am back and fully dedicated to pushing [OSS](https://github.com/Byron/bcore) forward in the realm of [digital content creation](http://gooseberry.blender.org/), and git-python will see some of my time as well. Therefore it will be moving forward, slowly but steadily. + +In short, I want to make a new release of 0.3 with all contributions and fixes included, foster community building to facilitate contributions. Everything else is future. + +#### PRESENT GOALS + +The goals I have set for myself, in order, are as follows, all on branch 0.3. + +* bring the test suite back online to work with the most commonly used git version +* setup a travis test-matrix to test against a lower and upper git version as well +* merge all open pull requests, may there be a test-case or not, back. If something breaks, fix it if possible or let the contributor know +* conform git-python's structure and toolchain to the one used in my [other OSS projects](https://github.com/Byron/bcore) +* evaluate all open issues and close them if possible +* create a new release of the 0.3 branch +* evaluate python 3.3 compatibility and establish it if possible + +While that is happening, I will try hard to foster community around the project. This means being more responsive on the mailing list and in issues, as well as setting up clear guide lines about the [contribution](http://rfc.zeromq.org/spec:22) and maintenance workflow. + +#### FUTURE GOALS + +There has been a lot of work in the master branch, which is the direction I want git-python to go. Namely, it should be able to freely mix and match the back-end used, depending on your requirements and environment. + +* restructure master to match my [OSS standard](https://github.com/Byron/bcore) +* review code base and bring test-suite back online +* establish python 3.3 compatibility +* make it work similarly to 0.3, but with the option to swap for at least one additional backend +* make a 1.0 release +* add backends as required + ### SOURCE From 3ea450112501e0d9f11e554aaf6ce9f36b32b732 Mon Sep 17 00:00:00 2001 From: s1341 Date: Fri, 9 May 2014 18:17:02 +0300 Subject: [PATCH 0244/3719] Fix typo in untracked_files --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index 977801052..8191b3057 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -535,7 +535,7 @@ def untracked_files(self): for line in proc.stdout: if not line.startswith(prefix): continue - filename = line[len(preffix):].rstrip('\n') + filename = line[len(prefix):].rstrip('\n') # Special characters are escaped if filename[0] == filename[-1] == '"': filename = filename[1:-1].decode('string_escape') From 2d3b5a303a65d6f80b84e63a1b3cf4b670c81f9a Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 15:38:03 +1000 Subject: [PATCH 0245/3719] Initial work for supporting python 3 (>= 3.3). Signed-off-by: David Black --- smmap/__init__.py | 4 ++-- smmap/buf.py | 8 ++++---- smmap/mman.py | 20 +++++++++++--------- smmap/test/test_buf.py | 6 +++--- smmap/test/test_mman.py | 8 ++++---- smmap/test/test_tutorial.py | 2 +- smmap/test/test_util.py | 8 ++++---- smmap/util.py | 13 ++++++++++--- 8 files changed, 39 insertions(+), 30 deletions(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index a10cd5c99..879ebea24 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -7,5 +7,5 @@ __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience -from mman import * -from buf import * +from .mman import * +from .buf import * diff --git a/smmap/buf.py b/smmap/buf.py index 255c6b54d..3917ee8be 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import WindowCursor +from .mman import WindowCursor import sys @@ -21,7 +21,7 @@ class SlidingWindowMapBuffer(object): ) - def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): """Initalize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access If None, you have call begin_access before using the buffer and provide a cursor @@ -61,7 +61,7 @@ def __getslice__(self, i, j): assert c.is_valid() if i < 0: i = self._size + i - if j == sys.maxint: + if j == sys.maxsize: j = self._size if j < 0: j = self._size + j @@ -86,7 +86,7 @@ def __getslice__(self, i, j): # END fast or slow path #{ Interface - def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. diff --git a/smmap/mman.py b/smmap/mman.py index 97c42c5bb..9cc251f0f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,15 +1,17 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -from util import ( +from .util import ( MapWindow, MapRegion, MapRegionList, is_64_bit, - align_to_mmap + align_to_mmap, + string_types, ) from weakref import ref import sys from sys import getrefcount +from functools import reduce __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] #{ Utilities @@ -218,7 +220,7 @@ def fd(self): **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), basestring): + if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() @@ -256,7 +258,7 @@ class StaticWindowMapManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): """initialize the manager with the given parameters. :param window_size: if -1, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size @@ -306,7 +308,7 @@ def _collect_lru_region(self, size): while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None lru_list = None - for regions in self._fdict.itervalues(): + for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and @@ -343,7 +345,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): r = a[0] else: try: - r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxsize, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -405,7 +407,7 @@ def num_file_handles(self): def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) def window_size(self): """:return: size of each window when allocating new regions""" @@ -445,7 +447,7 @@ def force_map_handle_removal_win(self, base_path): #END early bailout num_closed = 0 - for path, rlist in self._fdict.iteritems(): + for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: region._mf.close() @@ -473,7 +475,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): __slots__ = tuple() - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxsize): """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 4bdcb76f5..d40da1479 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager from smmap.buf import * @@ -22,8 +22,8 @@ def test_basics(self): # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 46429a419..0929583c9 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.mman import * from smmap.mman import WindowCursor @@ -66,7 +66,7 @@ def test_memory_manager(self): man._collect_lru_region(10) # doesn't fail if we overallocate - assert man._collect_lru_region(sys.maxint) == 0 + assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") @@ -80,9 +80,9 @@ def test_memory_manager(self): assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) + self.assertRaises(ValueError, c.path) else: - self.failUnlessRaises(ValueError, c.fd) + self.assertRaises(ValueError, c.fd) #END handle value error #END for each input os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index 4e1a5764b..ad1a9c0b5 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,4 +1,4 @@ -from lib import TestBase +from .lib import TestBase class TestTutorial(TestBase): diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 2df0660be..a009bd9d2 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.util import * @@ -38,7 +38,7 @@ def test_window(self): assert wc.ofs == 1 and wc.size == maxsize # without maxsize - wc.extend_right_to(wr, sys.maxint) + wc.extend_right_to(wr, sys.maxsize) assert wc.ofs_end() == wr.ofs and wc.ofs == 1 # extend left @@ -46,7 +46,7 @@ def test_window(self): wr.extend_left_to(wc2, maxsize) assert wr.size == maxsize - wr.extend_left_to(wc2, sys.maxint) + wr.extend_left_to(wc2, sys.maxsize) assert wr.ofs == wc2.ofs_end() wc.align() @@ -68,7 +68,7 @@ def test_region(self): assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. # We only test on linux as it is inconsitent between the python versions diff --git a/smmap/util.py b/smmap/util.py index c6710b3fe..fee9ad591 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -19,6 +19,13 @@ #{ Utilities +def string_types(): + if sys.version_info[0] >= 3: + return str + else: + return basestring + + def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. @@ -34,7 +41,7 @@ def align_to_mmap(num, round_up): def is_64_bit(): """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxint > (1<<32) - 1 + return sys.maxsize > (1<<32) - 1 #}END utilities @@ -154,7 +161,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping finally: - if isinstance(path_or_fd, basestring): + if isinstance(path_or_fd, string_types()): os.close(fd) #END only close it if we opened it #END close file handle @@ -258,7 +265,7 @@ def path_or_fd(self): def file_size(self): """:return: size of file we manager""" if self._file_size is None: - if isinstance(self._path_or_fd, basestring): + if isinstance(self._path_or_fd, string_types()): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size From 575f4a895ea525ecfedbfc1dc4c6f032ad92ccdc Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 16:35:20 +1000 Subject: [PATCH 0246/3719] instead of writing a string to a buffer api (in python 3) write a buffer. Signed-off-by: David Black --- smmap/test/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 21e6c5a09..01f6cc918 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -22,7 +22,7 @@ def __init__(self, size, prefix=''): fp = open(self._path, "wb") fp.seek(size-1) - fp.write('1') + fp.write(b'1') fp.close() assert os.path.getsize(self.path) == size From ef3b36e6ae20e7bd6c31c71623a5e0019ba9eec9 Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 16:36:47 +1000 Subject: [PATCH 0247/3719] _need_compat_layer is not required in python 3. Signed-off-by: David Black --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index fee9ad591..f4ecd228c 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -104,7 +104,7 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[1] < 6 + _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset From fe311b917b3cef75189e835bbef5eebd5b76cc20 Mon Sep 17 00:00:00 2001 From: "Derek D. Fedel" Date: Fri, 16 May 2014 08:00:29 -0700 Subject: [PATCH 0248/3719] Fix for #142. Simply ignores lines that begin with ' =' --- git/remote.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/git/remote.py b/git/remote.py index f89e9d83c..becfbd25b 100644 --- a/git/remote.py +++ b/git/remote.py @@ -513,14 +513,15 @@ def update(self, **kwargs): def _get_fetch_info_from_stderr(self, proc, progress): # skip first line as it is some remote info we are not interested in output = IterableList('name') - - + + # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = list() for line in digest_process_messages(proc.stderr, progress): - if line.startswith('From') or line.startswith('remote: Total') or line.startswith('POST'): + if line.startswith('From') or line.startswith('remote: Total') or line.startswith('POST') \ + or line.startswith(' ='): continue elif line.startswith('warning:'): print >> sys.stderr, line From afcf97e6cd6584c97ccfe3de57bdc58df0ba9f8a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 16 May 2014 08:31:09 -0700 Subject: [PATCH 0249/3719] Add tox.ini; make .travis.yml use it --- .gitignore | 1 + .travis.yml | 16 +++++++++------- tox.ini | 11 +++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 6cfb58df1..11852be02 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build/ coverage dist/ MANIFEST +.tox diff --git a/.travis.yml b/.travis.yml index d8640a08e..91f43283b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ language: python -python: - - "2.6" - - "2.7" - - "3.3" - # - "pypy" - no getrefcount - -script: nosetests +env: + - TOXENV=py26 + - TOXENV=py27 + - TOXENV=py33 + - TOXENV=py34 +install: + - pip install tox +script: + - tox diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..80190d0a3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py33, py34 + +[testenv] +commands = nosetests +deps = nose From 13e3a809554706905418a48b72e09e2eba81af2d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 19 May 2014 23:42:30 +0200 Subject: [PATCH 0250/3719] Added coverage report --- .coveragerc | 10 ++++++++++ .gitignore | 2 ++ .travis.yml | 5 ++++- README.md | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..410ffc520 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = git + +; to make nosetests happy +[report] +omit = + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index eec80860b..df821cfa8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.swp *~ /lib/GitPython.egg-info +cover/ +.coverage /build /dist /doc/_build diff --git a/.travis.yml b/.travis.yml index 2a56beccf..0a2906dc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,8 @@ python: install: - git submodule update --init --recursive - git fetch --tags + - pip install coveralls script: - - nosetests + - nosetests --with-coverage +# after_success: as long as we are not running smoothly ... give it the cover treatment every time + - coveralls diff --git a/README.md b/README.md index d978917bc..818e37515 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ A distribution package can be obtained for manual installation at: ### DEVELOPMENT STATUS [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) +[![Coverage Status](https://coveralls.io/repos/gitpython-developers/GitPython/badge.png)](https://coveralls.io/r/gitpython-developers/GitPython) The project was idle for 2 years, the last release was made about 3 years ago. Reason for this might have been the project's dependency on me as sole active maintainer, which is an issue in itself. From f0680101739da3435bbae0139765bb5ce65bcb92 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 19 May 2014 23:51:50 +0200 Subject: [PATCH 0251/3719] Added coverage reporting. In the process, I removed tox as it made things so much more complex for me. --- .coveragerc | 10 ++++++ .gitignore | 1 + .travis.yml | 15 ++++----- README.rst => README.md | 70 ++++++++++++++++++++++------------------- tox.ini | 11 ------- 5 files changed, 57 insertions(+), 50 deletions(-) create mode 100644 .coveragerc rename README.rst => README.md (71%) delete mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..e61d27ba9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = smmap + +; to make nosetests happy +[report] +omit = + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 11852be02..2081aafd2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ .coverage coverage +cover/ dist/ MANIFEST .tox diff --git a/.travis.yml b/.travis.yml index 91f43283b..c63e5e325 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ language: python -env: - - TOXENV=py26 - - TOXENV=py27 - - TOXENV=py33 - - TOXENV=py34 +python: + - "2.6" + - "2.7" + - "3.3" install: - - pip install tox + - pip install coveralls script: - - tox + - nosetests --with-coverage +after_success: + - coveralls diff --git a/README.rst b/README.md similarity index 71% rename from README.rst rename to README.md index 84feb373c..c056ed1c1 100644 --- a/README.rst +++ b/README.md @@ -1,15 +1,15 @@ -########### -Motivation -########### +## Motivation + When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. -######## -Overview -######## -.. image:: https://travis-ci.org/Byron/smmap.svg?branch=master :target: https://travis-ci.org/Byron/smmap + +## Overview + +[![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) +[![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. @@ -21,42 +21,49 @@ Although the library can be used most efficiently with its native interface, a B For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. -############# -Prerequisites -############# + + +## Prerequisites + * Python 2.4, 2.5 or 2.6 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. -########### -Limitations -########### + + +## Limitations + * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. * It wasn't tested on python 2.7 and 3.x. -################ -Installing smmap -################ -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + +## Installing smmap + +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively: - $ easy_install smmap - # or - $ pip install smmap +```bash +$ easy_install smmap +# or +$ pip install smmap +``` As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. -If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: +If you have downloaded the source archive, the package can be installed by running the `setup.py` script: - $ python setup.py install +```bash +$ python setup.py install +``` + +It is advised to have a look at the **Usage Guide** for a brief introduction on the different database implementations. + -It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. -################## -Homepage and Links -################## -The project is home on github at `https://github.com/Byron/smmap `_. +## Homepage and Links + +The project is home on github at https://github.com/Byron/smmap . The latest source can be cloned from github as well: @@ -72,10 +79,9 @@ Issues can be filed on github: * https://github.com/Byron/smmap/issues -################### -License Information -################### + + +## License Information + *smmap* is licensed under the New BSD License. -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools -.. _pip: http://www.pip-installer.org/en/latest/ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 80190d0a3..000000000 --- a/tox.ini +++ /dev/null @@ -1,11 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py26, py27, py33, py34 - -[testenv] -commands = nosetests -deps = nose From 4bc91d7495d7eb70a70c1f025137718f41486cd2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 4 Jun 2014 10:06:34 +0200 Subject: [PATCH 0252/3719] HACK: Removed assertion just to be a bit less annoyed by constant fail --- git/remote.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index becfbd25b..37ddd91bf 100644 --- a/git/remote.py +++ b/git/remote.py @@ -537,7 +537,10 @@ def _get_fetch_info_from_stderr(self, proc, progress): fetch_head_info = fp.readlines() fp.close() - assert len(fetch_info_lines) == len(fetch_head_info), "len(%s) != len(%s)" % (fetch_head_info, fetch_info_lines) + # NOTE: HACK Just disabling this line will make github repositories work much better. + # I simply couldn't stand it anymore, so here is the quick and dirty fix ... . + # This project needs a lot of work ! + # assert len(fetch_info_lines) == len(fetch_head_info), "len(%s) != len(%s)" % (fetch_head_info, fetch_info_lines) output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line,fetch_line in zip(fetch_info_lines, fetch_head_info)) From e8980057ccfcaca34b423804222a9f981350ac67 Mon Sep 17 00:00:00 2001 From: Marios Zindilis Date: Fri, 13 Jun 2014 00:16:50 +0300 Subject: [PATCH 0253/3719] Changed link to PyPI --- doc/source/intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 520cf159a..8dac28047 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -47,7 +47,7 @@ here: * `setuptools`_ * `install setuptools `_ -* `pypi `_ +* `pypi `_ .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools From eb53329253f8ec1d0eff83037ce70260b6d8fcce Mon Sep 17 00:00:00 2001 From: Marios Zindilis Date: Fri, 13 Jun 2014 00:51:04 +0300 Subject: [PATCH 0254/3719] Fixed two minor typos. --- git/repo/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 8191b3057..71492fe87 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -56,13 +56,13 @@ class Repo(object): The following attributes are worth using: - 'working_dir' is the working directory of the git command, wich is the working tree + 'working_dir' is the working directory of the git command, which is the working tree directory if available or the .git directory in case of bare repositories 'working_tree_dir' is the working tree directory, but will raise AssertionError if we are a bare repository. - 'git_dir' is the .git repository directoy, which is always set.""" + 'git_dir' is the .git repository directory, which is always set.""" DAEMON_EXPORT_FILE = 'git-daemon-export-ok' __slots__ = ( "working_dir", "_working_tree_dir", "git_dir", "_bare", "git", "odb" ) From 403ad7816e2c88be78606cfee2d9944f535e7ed7 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 13 Jun 2014 22:24:23 -0700 Subject: [PATCH 0255/3719] Delay importing sys.getrefcount until needed This makes it possibly to at least install on PyPy Addresses: GH-4 ("pypy compatibility") --- smmap/mman.py | 1 - smmap/util.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9cc251f0f..637a2844d 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -10,7 +10,6 @@ from weakref import ref import sys -from sys import getrefcount from functools import reduce __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] diff --git a/smmap/util.py b/smmap/util.py index f4ecd228c..ec86cbf16 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,8 +12,6 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -from sys import getrefcount - __all__ = [ "align_to_mmap", "is_64_bit", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] @@ -210,6 +208,7 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" + from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 @@ -256,6 +255,7 @@ def __init__(self, path_or_fd): def client_count(self): """:return: amount of clients which hold a reference to this instance""" + from sys import getrefcount return getrefcount(self)-3 def path_or_fd(self): From f3d3502bef512ed48581f69fee9b655a91e06db8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Sun, 15 Jun 2014 23:33:04 -0700 Subject: [PATCH 0256/3719] Change / to // (integer division) in several places This fixes a bunch of bugs and test failures in Python 3, which uses "true division" for / --- smmap/test/test_mman.py | 6 +++--- smmap/test/test_util.py | 2 +- smmap/util.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 0929583c9..e0516b21c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -95,8 +95,8 @@ def test_memman_operation(self): fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size @@ -110,7 +110,7 @@ def test_memman_operation(self): base_offset = 5000 # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() / 2 + size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() assert rr().client_count() == 2 # the manager and the cursor and us diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index a009bd9d2..8afba005e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -54,7 +54,7 @@ def test_window(self): def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size / 2 + half_size = fc.size // 2 rofs = align_to_mmap(4200, False) rfull = MapRegion(fc.path, 0, fc.size) rhalfofs = MapRegion(fc.path, rofs, fc.size) diff --git a/smmap/util.py b/smmap/util.py index ec86cbf16..0d8385db2 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -31,7 +31,7 @@ def align_to_mmap(num, round_up): :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; if round_up and (res != num): res += ALLOCATIONGRANULARITY #END handle size From 707885bebe6cefa45dae7dd13dadac5b2b98d16d Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:41:25 -0700 Subject: [PATCH 0257/3719] .gitignore: Add *.egg-info --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2081aafd2..73b0b6188 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ cover/ dist/ MANIFEST .tox +*.egg-info From 23b79ea2838510028de43a6edb1b09246dca9a46 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:47:31 -0700 Subject: [PATCH 0258/3719] Add back tox.ini for tox This time with support for measuring coverage. Ability to test quickly across multiple Python versions is crucial for working on Python 3 compatibility... --- tox.ini | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..e0e196418 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py33, py34 + +[testenv] +commands = nosetests {posargs:--with-coverage --cover-package=smmap} +deps = + nose + nosexcover From 52fba74d78d3fcce53fb89c7b2c20b24197296a8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:33:41 -0700 Subject: [PATCH 0259/3719] Deal with lack of `buffer` in py3 --- smmap/mman.py | 1 + smmap/test/test_tutorial.py | 1 + smmap/util.py | 11 ++++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/smmap/mman.py b/smmap/mman.py index 637a2844d..ff32bbb51 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -6,6 +6,7 @@ is_64_bit, align_to_mmap, string_types, + buffer, ) from weakref import ref diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index ad1a9c0b5..ccc113b4f 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -44,6 +44,7 @@ def test_example(self): # its recommended not to create big slices when feeding the buffer # into consumers (e.g. struct or zlib). # Instead, either give the buffer directly, or use pythons buffer command. + from smmap.util import buffer buffer(c.buffer(), 1, 9) # first 9 bytes without copying them # you can query absolute offsets, and check whether an offset is included diff --git a/smmap/util.py b/smmap/util.py index 0d8385db2..54c6c45c3 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,11 +12,20 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -__all__ = [ "align_to_mmap", "is_64_bit", +__all__ = [ "align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities +try: + # Python 2 + buffer = buffer +except NameError: + # Python 3 has no `buffer`; only `memoryview` + def buffer(obj, offset, size): + return memoryview(obj[offset:offset+size]) + + def string_types(): if sys.version_info[0] >= 3: return str From cf515928248b7c34cb73b6e20edbdadebd086f96 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 07:55:21 -0700 Subject: [PATCH 0260/3719] Fix 2 instances of "containnig" => "containing" --- smmap/mman.py | 2 +- smmap/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ff32bbb51..7cbb535bd 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,4 +1,4 @@ -"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( MapWindow, MapRegion, diff --git a/smmap/util.py b/smmap/util.py index 54c6c45c3..c37dfdd31 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -1,4 +1,4 @@ -"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" import os import sys import mmap From c0b94c67b86b4250364a951f4865d714952f792a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 08:54:30 -0700 Subject: [PATCH 0261/3719] .travis.yml: Allow py33 to fail, add py34, etc. --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c63e5e325..d02eb6f5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,17 @@ language: python python: - - "2.6" - - "2.7" - - "3.3" + - 2.6 + - 2.7 + - 3.3 + - 3.4 install: - pip install coveralls script: - nosetests --with-coverage after_success: - coveralls +matrix: + allow_failures: + - python: 3.3 + - python: 3.4 + fast_finish: true From 33e6314941572a93b8031b1feecded22ef1f5f3c Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 08:49:04 -0700 Subject: [PATCH 0262/3719] Use bytes() instead of str() bytes() is more accurate and is actually correct in Python 3, whereas str() is incorrect in Python 3, because it's a Unicode string. --- smmap/buf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/smmap/buf.py b/smmap/buf.py index 3917ee8be..ba6f8ede7 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -5,6 +5,12 @@ __all__ = ["SlidingWindowMapBuffer"] +try: + bytes +except NameError: + bytes = str + + class SlidingWindowMapBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. @@ -73,7 +79,7 @@ def __getslice__(self, i, j): ofs = i # Keeping tokens in a list could possible be faster, but the list # overhead outweighs the benefits (tested) ! - md = str() + md = bytes() while l: c.use_region(ofs, l) assert c.is_valid() From 19917bdc3d30d7d3e6dba082ba132a86d0190b09 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 09:05:33 -0700 Subject: [PATCH 0263/3719] Fix typo: "optimial" => "optimal" --- smmap/test/test_buf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d40da1479..23a4fbbcd 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -76,7 +76,7 @@ def test_basics(self): for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case'), - (static_man, 'static optimial')): + (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi From 8fde5f3fb8711f866612c657d20c9d3e9cf59f41 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 09:02:30 -0700 Subject: [PATCH 0264/3719] Make __getitem__ handle slice for Python 3 Python 3 doesn't have __getslice__ instead it uses __getitem__ with a slice object. --- smmap/buf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smmap/buf.py b/smmap/buf.py index ba6f8ede7..2f27d4d01 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -51,6 +51,8 @@ def __len__(self): return self._size def __getitem__(self, i): + if isinstance(i, slice): + return self.__getslice__(i.start or 0, i.stop or self._size) c = self._c assert c.is_valid() if i < 0: From 651dfa8a359200b66e686998af2640efb54f16ad Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 10:48:43 -0700 Subject: [PATCH 0265/3719] Change / to // (integer division) in test_buf.py This fixes (the last!) test failure in Python 3, which uses "true division" for / --- smmap/test/test_buf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 23a4fbbcd..15dfb8238 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -10,9 +10,10 @@ man_optimal = SlidingWindowMapManager() -man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, - max_memory_size=TestBase.k_window_test_size/3, - max_open_handles=15) +man_worst_case = SlidingWindowMapManager( + window_size=TestBase.k_window_test_size // 100, + max_memory_size=TestBase.k_window_test_size // 3, + max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): From f75dcbcf5d10a639c031dd706fa7fcfa1784eecb Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 11:09:54 -0700 Subject: [PATCH 0266/3719] .travis.yml: Stop allowing failures for py3{3,4} --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d02eb6f5e..47cb41170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,3 @@ script: - nosetests --with-coverage after_success: - coveralls -matrix: - allow_failures: - - python: 3.3 - - python: 3.4 - fast_finish: true From fbbb3090ea1ca4ac3042acea7633241b0388f0b3 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 11:31:48 -0700 Subject: [PATCH 0267/3719] setup.py: Add Python 3 (and Python 2) classifiers --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 2e97e59c6..13d2063ab 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,12 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", ], long_description=long_description, ) From 55267119140f3828a24b4986600ed21a1808d6cc Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 12:02:10 -0700 Subject: [PATCH 0268/3719] setup.cfg: Specify that wheel is universal --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From d9fc8b6c06b91dfddb73d18eaa8164e64cc2600a Mon Sep 17 00:00:00 2001 From: William Gibb Date: Thu, 26 Jun 2014 11:21:55 -0400 Subject: [PATCH 0269/3719] Add patch from to 0.3 branch. https://github.com/gitpython-developers/GitPython/commit/f362d10fa24395c21b1629923ccd705ba73ae996 Related to #43 --- git/util.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/git/util.py b/git/util.py index 7c257b37c..88a72c0cb 100644 --- a/git/util.py +++ b/git/util.py @@ -22,6 +22,10 @@ to_bin_sha ) +# Import the user database on unix based systems +if os.name == "posix": + import pwd + __all__ = ( "stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', @@ -113,12 +117,17 @@ def assure_directory_exists(path, is_file=False): def get_user_id(): """:return: string identifying the currently active system user as name@node - :note: user can be set with the 'USER' environment variable, usually set on windows""" - ukn = 'UNKNOWN' - username = os.environ.get('USER', os.environ.get('USERNAME', ukn)) - if username == ukn and hasattr(os, 'getlogin'): - username = os.getlogin() - # END get username from login + :note: user can be set with the 'USER' environment variable, usually set on windows + :note: on unix based systems you can use the password database + to get the login name of the effective process user""" + if os.name == "posix": + username = pwd.getpwuid(os.geteuid()).pw_name + else: + ukn = 'UNKNOWN' + username = os.environ.get('USER', os.environ.get('USERNAME', ukn)) + if username == ukn and hasattr(os, 'getlogin'): + username = os.getlogin() + # END get username from login return "%s@%s" % (username, platform.node()) #} END utilities From 237d47b9d714fcc2eaedff68c6c0870ef3e0041a Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 3 Jul 2014 18:46:45 +1200 Subject: [PATCH 0270/3719] Support multiple refspecs in fetch. Git supports fetching many refs at once - support this in GitPython too for more efficient operations when selectively mirroring repositories. --- git/remote.py | 10 +++++++++- git/test/test_remote.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 37ddd91bf..b06c0686b 100644 --- a/git/remote.py +++ b/git/remote.py @@ -583,6 +583,10 @@ def fetch(self, refspec=None, progress=None, **kwargs): See also git-push(1). Taken from the git manual + + Fetch supports multiple refspecs (as the + underlying git-fetch does) - supplying a list rather than a string + for 'refspec' will make use of this facility. :param progress: See 'push' method :param kwargs: Additional arguments to be passed to git-fetch :return: @@ -593,7 +597,11 @@ def fetch(self, refspec=None, progress=None, **kwargs): As fetch does not provide progress information to non-ttys, we cannot make it available here unfortunately as in the 'push' method.""" kwargs = add_progress(kwargs, self.repo.git, progress) - proc = self.repo.git.fetch(self, refspec, with_extended_output=True, as_process=True, v=True, **kwargs) + if isinstance(refspec, list): + args = refspec + else: + args = [refspec] + proc = self.repo.git.fetch(self, *args, with_extended_output=True, as_process=True, v=True, **kwargs) return self._get_fetch_info_from_stderr(proc, progress or RemoteProgress()) def pull(self, refspec=None, progress=None, **kwargs): diff --git a/git/test/test_remote.py b/git/test/test_remote.py index a7f1be22e..b12480965 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -199,6 +199,10 @@ def get_info(res, remote, name): # ... with respec and no target res = fetch_and_test(remote, refspec='master') assert len(res) == 1 + + # ... multiple refspecs + res = fetch_and_test(remote, refspec=['master', 'fred']) + assert len(res) == 1 # add new tag reference rtag = TagReference.create(remote_repo, "1.0-RV_hello.there") From d79c655a7bf281a39662f4a4562d76312b69515a Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 14:21:09 -0400 Subject: [PATCH 0271/3719] Update async to the Python 2 / 3 compatible version --- gitdb/ext/async | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/async b/gitdb/ext/async index 90326fb86..339024bfb 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 +Subproject commit 339024bfb1d0a2b091e63d7a7ea23a1c63189f5c From 72167492334b756ddeaa606274e9348a70734cdb Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 14:27:22 -0400 Subject: [PATCH 0272/3719] Update smmap to a Python 3 compatible version --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 616e9ceaf..552671191 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd +Subproject commit 55267119140f3828a24b4986600ed21a1808d6cc From 1c6f4c19289732bd13507eba9e54c9d692957137 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:35:24 -0400 Subject: [PATCH 0273/3719] Automated PEP 8 fixes --- gitdb/__init__.py | 4 +- gitdb/base.py | 205 ++++++++++--------- gitdb/db/base.py | 162 ++++++++------- gitdb/db/git.py | 34 ++-- gitdb/db/loose.py | 103 +++++----- gitdb/db/mem.py | 56 +++-- gitdb/db/pack.py | 88 ++++---- gitdb/db/ref.py | 18 +- gitdb/exc.py | 10 +- gitdb/fun.py | 230 ++++++++++----------- gitdb/pack.py | 394 ++++++++++++++++++------------------ gitdb/stream.py | 318 +++++++++++++++-------------- gitdb/test/__init__.py | 2 +- gitdb/test/db/lib.py | 80 ++++---- gitdb/test/db/test_git.py | 22 +- gitdb/test/db/test_loose.py | 15 +- gitdb/test/db/test_mem.py | 18 +- gitdb/test/db/test_pack.py | 28 +-- gitdb/test/db/test_ref.py | 28 ++- gitdb/test/lib.py | 39 ++-- gitdb/test/test_base.py | 28 +-- gitdb/test/test_example.py | 26 +-- gitdb/test/test_pack.py | 102 +++++----- gitdb/test/test_stream.py | 51 +++-- gitdb/test/test_util.py | 43 ++-- gitdb/util.py | 131 ++++++------ 26 files changed, 1108 insertions(+), 1127 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ff750d14c..847269a33 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -12,14 +12,14 @@ def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('async', 'smmap'): sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - + try: __import__(module) except ImportError: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) #END verify import #END handel imports - + #} END initialization _init_externals() diff --git a/gitdb/base.py b/gitdb/base.py index bad5f7472..a673c2376 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -13,183 +13,183 @@ type_to_type_id_map ) -__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', 'IStream', 'InvalidOInfo', 'InvalidOStream' ) #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provding information + """Carries information about an object in an ODB, provding information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. - + It can be accessed using tuple notation and using attribute access notation:: - + assert dbi[0] == dbi.binsha assert dbi[1] == dbi.type assert dbi[2] == dbi.size - + The type is designed to be as lighteight as possible.""" __slots__ = tuple() - + def __new__(cls, sha, type, size): return tuple.__new__(cls, (sha, type, size)) - + def __init__(self, *args): tuple.__init__(self) - - #{ Interface + + #{ Interface @property def binsha(self): """:return: our sha as binary, 20 bytes""" return self[0] - + @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" return bin_to_hex(self[0]) - + @property def type(self): return self[1] - + @property def type_id(self): return type_to_type_id_map[self[1]] - + @property def size(self): return self[2] #} END interface - - + + class OPackInfo(tuple): - """As OInfo, but provides a type_id property to retrieve the numerical type id, and + """As OInfo, but provides a type_id property to retrieve the numerical type id, and does not include a sha. - - Additionally, the pack_offset is the absolute offset into the packfile at which + + Additionally, the pack_offset is the absolute offset into the packfile at which all object information is located. The data_offset property points to the abosolute location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size): return tuple.__new__(cls, (packoffset,type, size)) - + def __init__(self, *args): tuple.__init__(self) - - #{ Interface - + + #{ Interface + @property def pack_offset(self): return self[0] - + @property def type(self): return type_id_to_type_map[self[1]] - + @property def type_id(self): return self[1] - + @property def size(self): return self[2] - + #} END interface - - + + class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, or the negative offset from the pack_offset, so that pack_offset - delta_info yields the pack offset of the base object""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, delta_info): return tuple.__new__(cls, (packoffset, type, size, delta_info)) - - #{ Interface + + #{ Interface @property def delta_info(self): return self[3] - #} END interface - - + #} END interface + + class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional + """Base for object streams retrieved from the database, providing additional information about the stream. Generally, ODB streams are read-only as objects are immutable""" __slots__ = tuple() - + def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - - + + def __init__(self, *args, **kwargs): tuple.__init__(self) - - #{ Stream Reader Interface - + + #{ Stream Reader Interface + def read(self, size=-1): return self[3].read(size) - + @property def stream(self): return self[3] - + #} END stream reader interface - - + + class ODeltaStream(OStream): """Uses size info of its stream, delaying reads""" - + def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - + #{ Stream Reader Interface - + @property def size(self): return self[3].size - + #} END stream reader interface - - + + class OPackStream(OPackInfo): """Next to pack object information, a stream outputting an undeltified base object is provided""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (packoffset, type, size, stream)) - - #{ Stream Reader Interface + + #{ Stream Reader Interface def read(self, size=-1): return self[3].read(size) - + @property def stream(self): return self[3] #} END stream reader interface - + class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, delta_info, stream): return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface + #{ Stream Reader Interface def read(self, size=-1): return self[4].read(size) - + @property def stream(self): return self[4] @@ -197,106 +197,106 @@ def stream(self): class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow + """Represents an input content stream to be fed into the ODB. It is mutable to allow the ODB to record information about the operations outcome right in this instance. - + It provides interfaces for the OStream and a StreamReader to allow the instance to blend in without prior conversion. - + The only method your content stream must support is 'read'""" __slots__ = tuple() - + def __new__(cls, type, size, stream, sha=None): return list.__new__(cls, (sha, type, size, stream, None)) - + def __init__(self, type, size, stream, sha=None): list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface + + #{ Interface @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" return bin_to_hex(self[0]) - + def _error(self): """:return: the error that occurred when processing the stream, or None""" return self[4] - + def _set_error(self, exc): """Set this input stream to the given exc, may be None to reset the error""" self[4] = exc - + error = property(_error, _set_error) - + #} END interface - + #{ Stream Reader Interface - + def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on + """Implements a simple stream reader interface, passing the read call on to our internal stream""" return self[3].read(size) - - #} END stream reader interface - + + #} END stream reader interface + #{ interface - + def _set_binsha(self, binsha): self[0] = binsha - + def _binsha(self): return self[0] - + binsha = property(_binsha, _set_binsha) - - + + def _type(self): return self[1] - + def _set_type(self, type): self[1] = type - + type = property(_type, _set_type) - + def _size(self): return self[2] - + def _set_size(self, size): self[2] = size - + size = property(_size, _set_size) - + def _stream(self): return self[3] - + def _set_stream(self, stream): self[3] = stream - + stream = property(_stream, _set_stream) - - #} END odb info interface - + + #} END odb info interface + class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in + """Carries information about a sha identifying an object which is invalid in the queried database. The exception attribute provides more information about the cause of the issue""" __slots__ = tuple() - + def __new__(cls, sha, exc): return tuple.__new__(cls, (sha, exc)) - + def __init__(self, sha, exc): tuple.__init__(self, (sha, exc)) - + @property def binsha(self): return self[0] - + @property def hexsha(self): return bin_to_hex(self[0]) - + @property def error(self): """:return: exception instance explaining the failure""" @@ -306,6 +306,5 @@ def error(self): class InvalidOStream(InvalidOInfo): """Carries information about an invalid ODB stream""" __slots__ = tuple() - -#} END ODB Bases +#} END ODB Bases diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 867e93a81..0eef1e5d5 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,20 +4,20 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, - join, - LazyMixin, - hex_to_bin - ) + pool, + join, + LazyMixin, + hex_to_bin +) from gitdb.exc import ( - BadObject, - AmbiguousObjectName - ) + BadObject, + AmbiguousObjectName +) from async import ( - ChannelThreadTask - ) + ChannelThreadTask +) from itertools import chain @@ -28,17 +28,17 @@ class ObjectDBR(object): """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" - + def __contains__(self, sha): return self.has_obj - - #{ Query Interface + + #{ Query Interface def has_object(self, sha): """ :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") - + def has_object_async(self, reader): """Return a reader yielding information about the membership of objects as identified by shas @@ -46,62 +46,62 @@ def has_object_async(self, reader): :return: async.Reader yielding tuples of (sha, bool) pairs which indicate whether the given sha exists in the database or not""" task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - + return pool.add_task(task) + def info(self, sha): """ :return: OInfo instance :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - + def info_async(self, reader): """Retrieve information of a multitude of objects asynchronously :param reader: Channel yielding the sha's of the objects of interest :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" task = ChannelThreadTask(reader, str(self.info_async), self.info) return pool.add_task(task) - + def stream(self, sha): """:return: OStream instance :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - + def stream_async(self, reader): """Retrieve the OStream of multiple objects :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to + + **Note:** depending on the system configuration, it might not be possible to read all OStreams at once. Instead, read them individually using reader.read(x) where x is small enough.""" # base implementation just uses the stream method repeatedly task = ChannelThreadTask(reader, str(self.stream_async), self.stream) return pool.add_task(task) - + def size(self): """:return: amount of objects in this database""" raise NotImplementedError() - + def sha_iter(self): """Return iterator yielding 20 byte shas for all objects in this data base""" raise NotImplementedError() - + #} END query interface - - + + class ObjectDBW(object): """Defines an interface to create objects in the database""" - + def __init__(self, *args, **kwargs): self._ostream = None - + #{ Edit Interface def set_ostream(self, stream): """ Adjusts the stream to which all data should be sent when storing new objects - + :param stream: if not None, the stream to use, if None the default stream will be used. :return: previously installed stream, or None if there was no override @@ -109,96 +109,96 @@ def set_ostream(self, stream): cstream = self._ostream self._ostream = stream return cstream - + def ostream(self): """ :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream - + def store(self, istream): """ Create a new object in the database :return: the input istream object with its sha set to its corresponding value - - :param istream: IStream compatible instance. If its sha is already set - to a value, the object will just be stored in the our database format, + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, in which case the input stream is expected to be in object format ( header + contents ). :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - + def store_async(self, reader): """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as + Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as they are computed. - + :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, + The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. - + :param reader: async.Reader yielding IStream instances. The same instances will be used in the output channel as were received in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how + + **Note:** As some ODB implementations implement this operation atomic, they might + abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) + task = ChannelThreadTask(reader, str(self.store_async), self.store) return pool.add_task(task) - + #} END edit interface - + class FileDBBase(object): - """Provides basic facilities to retrieve files of interest, including + """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" - + def __init__(self, root_path): """Initialize this instance to look for its files at the given root path All subsequent operations will be relative to this path - :raise InvalidDBRoot: + :raise InvalidDBRoot: **Note:** The base will not perform any accessablity checking as the base - might not yet be accessible, but become accessible before the first + might not yet be accessible, but become accessible before the first access.""" super(FileDBBase, self).__init__() self._root_path = root_path - - - #{ Interface + + + #{ Interface def root_path(self): """:return: path at which this db operates""" return self._root_path - + def db_path(self, rela_path): """ - :return: the given relative path relative to our database root, allowing + :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" return join(self._root_path, rela_path) #} END interface - + class CachingDB(object): """A database which uses caches to speed-up access""" - - #{ Interface + + #{ Interface def update_cache(self, force=False): """ Call this method if the underlying data changed to trigger an update of the internal caching structures. - + :param force: if True, the update must be performed. Otherwise the implementation may decide not to perform an update if it thinks nothing has changed. :return: True if an update was performed as something change indeed""" - + # END interface def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed + """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" if isinstance(database, CompoundDB): compounds = list() @@ -209,11 +209,11 @@ def _databases_recursive(database, output): else: output.append(database) # END handle database type - + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): """A database which delegates calls to sub-databases. - + Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" def _set_cache_(self, attr): @@ -223,27 +223,27 @@ def _set_cache_(self, attr): self._db_cache = dict() else: super(CompoundDB, self)._set_cache_(attr) - + def _db_query(self, sha): """:return: database containing the given 20 byte sha :raise BadObject:""" - # most databases use binary representations, prevent converting + # most databases use binary representations, prevent converting # it everytime a database is being queried try: return self._db_cache[sha] except KeyError: pass # END first level cache - + for db in self._dbs: if db.has_object(sha): self._db_cache[sha] = db return db # END for each database raise BadObject(sha) - - #{ ObjectDBR interface - + + #{ ObjectDBR interface + def has_object(self, sha): try: self._db_query(sha) @@ -251,24 +251,24 @@ def has_object(self, sha): except BadObject: return False # END handle exceptions - + def info(self, sha): return self._db_query(sha).info(sha) - + def stream(self, sha): return self._db_query(sha).stream(sha) def size(self): """:return: total size of all contained databases""" return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) - + def sha_iter(self): return chain(*(db.sha_iter() for db in self._dbs)) - + #} END object DBR Interface - + #{ Interface - + def databases(self): """:return: tuple of database instances we use for lookups""" return tuple(self._dbs) @@ -283,7 +283,7 @@ def update_cache(self, force=False): # END if is caching db # END for each database to update return stat - + def partial_to_complete_sha_hex(self, partial_hexsha): """ :return: 20 byte binary sha1 from the given less-than-40 byte hexsha @@ -291,14 +291,14 @@ def partial_to_complete_sha_hex(self, partial_hexsha): :raise AmbiguousObjectName: """ databases = list() _databases_recursive(self, databases) - + len_partial_hexsha = len(partial_hexsha) if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") else: partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - + # END assure successful binary conversion + candidate = None for db in databases: full_bin_sha = None @@ -320,7 +320,5 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if not candidate: raise BadObject(partial_binsha) return candidate - - #} END interface - + #} END interface diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 1d6ad0f26..6e6ec5d1f 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -14,10 +14,11 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName +) + import os __all__ = ('GitDB', ) @@ -30,21 +31,21 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): PackDBCls = PackedDB LooseDBCls = LooseObjectDB ReferenceDBCls = ReferenceDB - + # Directories packs_dir = 'pack' loose_dir = '' alternates_dir = os.path.join('info', 'alternates') - + def __init__(self, root_path): """Initialize ourselves on a git objects directory""" super(GitDB, self).__init__(root_path) - + def _set_cache_(self, attr): if attr == '_dbs' or attr == '_loose_db': self._dbs = list() loose_db = None - for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), (self.loose_dir, self.LooseDBCls), (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) @@ -55,31 +56,30 @@ def _set_cache_(self, attr): # END remember loose db # END check path exists # END for each db type - + # should have at least one subdb if not self._dbs: raise InvalidDBRoot(self.root_path()) # END handle error - + # we the first one should have the store method assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" - + # finally set the value self._loose_db = loose_db else: super(GitDB, self)._set_cache_(attr) # END handle attrs - + #{ ObjectDBW interface - + def store(self, istream): return self._loose_db.store(istream) - + def ostream(self): return self._loose_db.ostream() - + def set_ostream(self, ostream): return self._loose_db.set_ostream(ostream) - + #} END objectdbw interface - diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index dc0ea0e3b..4ebca84d3 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -10,46 +10,46 @@ from gitdb.exc import ( - InvalidDBRoot, + InvalidDBRoot, BadObject, AmbiguousObjectName - ) +) from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, FDStream, Sha1Writer - ) +) from gitdb.base import ( - OStream, - OInfo - ) + OStream, + OInfo +) from gitdb.util import ( - file_contents_ro_filepath, - ENOENT, - hex_to_bin, - bin_to_hex, - exists, - chmod, - isdir, - isfile, - remove, - mkdir, - rename, - dirname, - basename, - join - ) - -from gitdb.fun import ( + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isdir, + isfile, + remove, + mkdir, + rename, + dirname, + basename, + join +) + +from gitdb.fun import ( chunk_size, - loose_object_header_info, + loose_object_header_info, write_object, stream_copy - ) +) import tempfile import mmap @@ -62,11 +62,11 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): """A database which operates on loose object files""" - + # CONFIGURATION # chunks in which data will be copied between streams stream_chunk_size = chunk_size - + # On windows we need to keep it writable, otherwise it cannot be removed # either new_objects_mode = 0444 @@ -81,14 +81,14 @@ def __init__(self, root_path): # Depending on the root, this might work for some mounts, for others not, which # is why it is per instance self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface + + #{ Interface def object_path(self, hexsha): """ - :return: path at which the object with the given hexsha would be stored, + :return: path at which the object with the given hexsha would be stored, relative to the database root""" return join(hexsha[:2], hexsha[2:]) - + def readable_db_object_path(self, hexsha): """ :return: readable object path to the object identified by hexsha @@ -97,8 +97,8 @@ def readable_db_object_path(self, hexsha): return self._hexsha_to_file[hexsha] except KeyError: pass - # END ignore cache misses - + # END ignore cache misses + # try filesystem path = self.db_path(self.object_path(hexsha)) if exists(path): @@ -106,11 +106,11 @@ def readable_db_object_path(self, hexsha): return path # END handle cache raise BadObject(hexsha) - + def partial_to_complete_sha_hex(self, partial_hexsha): """:return: 20 byte binary sha1 string which matches the given name uniquely :param name: hexadecimal partial name - :raise AmbiguousObjectName: + :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for binsha in self.sha_iter(): @@ -123,9 +123,9 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if candidate is None: raise BadObject(partial_hexsha) return candidate - + #} END interface - + def _map_loose_object(self, sha): """ :return: memory map of that file to allow random read access @@ -151,13 +151,13 @@ def _map_loose_object(self, sha): finally: os.close(fd) # END assure file is closed - + def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" if stream is not None and not isinstance(stream, Sha1Writer): raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) return super(LooseObjectDB, self).set_ostream(stream) - + def info(self, sha): m = self._map_loose_object(sha) try: @@ -166,12 +166,12 @@ def info(self, sha): finally: m.close() # END assure release of system resources - + def stream(self, sha): m = self._map_loose_object(sha) type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) return OStream(sha, type, size, stream) - + def has_object(self, sha): try: self.readable_db_object_path(bin_to_hex(sha)) @@ -179,7 +179,7 @@ def has_object(self, sha): except BadObject: return False # END check existance - + def store(self, istream): """note: The sha we produce will be hex by nature""" tmp_path = None @@ -187,14 +187,14 @@ def store(self, istream): if writer is None: # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - + if istream.binsha is None: writer = FDCompressedSha1Writer(fd) else: writer = FDStream(fd) # END handle direct stream copies # END handle custom writer - + try: try: if istream.binsha is not None: @@ -215,14 +215,14 @@ def store(self, istream): os.remove(tmp_path) raise # END assure tmpfile removal on error - + hexsha = None if istream.binsha: hexsha = istream.hexsha else: hexsha = writer.sha(as_hex=True) # END handle sha - + if tmp_path: obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) @@ -234,29 +234,28 @@ def store(self, istream): remove(obj_path) # END handle win322 rename(tmp_path, obj_path) - + # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr chmod(obj_path, self.new_objects_mode) # END handle dry_run - + istream.binsha = hex_to_bin(hexsha) return istream - + def sha_iter(self): # find all files which look like an object, extract sha from there for root, dirs, files in os.walk(self.root_path()): root_base = basename(root) if len(root_base) != 2: continue - + for f in files: if len(f) != 38: continue yield hex_to_bin(root_base + f) # END for each file # END for each walk iteration - + def size(self): return len(tuple(self.sha_iter())) - diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index b9b2b8995..e4fba94b3 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -10,18 +10,14 @@ ) from gitdb.base import ( - OStream, - IStream, - ) + OStream, + IStream, +) from gitdb.exc import ( - BadObject, - UnsupportedOperation - ) -from gitdb.stream import ( - ZippedStoreShaWriter, - DecompressMemMapReader, - ) + BadObject, + UnsupportedOperation +) from cStringIO import StringIO @@ -32,45 +28,45 @@ class MemoryDB(ObjectDBR, ObjectDBW): retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already exists in the target storage before introducing actual IO - + **Note:** memory is currently not threadsafe, hence the async methods cannot be used for storing""" - + def __init__(self): super(MemoryDB, self).__init__() self._db = LooseObjectDB("path/doesnt/matter") - + # maps 20 byte shas to their OStream objects self._cache = dict() - + def set_ostream(self, stream): raise UnsupportedOperation("MemoryDB's always stream into memory") - + def store(self, istream): zstream = ZippedStoreShaWriter() self._db.set_ostream(zstream) - + istream = self._db.store(istream) zstream.close() # close to flush zstream.seek(0) - - # don't provide a size, the stream is written in object format, hence the + + # don't provide a size, the stream is written in object format, hence the # header needs decompression - decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) - + return istream - + def store_async(self, reader): raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - + def has_object(self, sha): return sha in self._cache def info(self, sha): # we always return streams, which are infos as well return self.stream(sha) - + def stream(self, sha): try: ostream = self._cache[sha] @@ -80,15 +76,15 @@ def stream(self, sha): except KeyError: raise BadObject(sha) # END exception handling - + def size(self): return len(self._cache) - + def sha_iter(self): return self._cache.iterkeys() - - - #{ Interface + + + #{ Interface def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly @@ -100,12 +96,12 @@ def stream_copy(self, sha_iter, odb): if odb.has_object(sha): continue # END check object existance - + ostream = self.stream(sha) # compressed data including header sio = StringIO(ostream.stream.data()) istream = IStream(ostream.type, ostream.size, sio, sha) - + odb.store(istream) count += 1 # END for each sha diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 928731937..09f811847 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -12,10 +12,10 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - BadObject, - UnsupportedOperation, - AmbiguousObjectName - ) + BadObject, + UnsupportedOperation, + AmbiguousObjectName +) from gitdb.pack import PackEntity @@ -29,12 +29,12 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): """A database operating on a set of object packs""" - + # sort the priority list every N queries - # Higher values are better, performance tests don't show this has + # Higher values are better, performance tests don't show this has # any effect, but it should have one _sort_interval = 500 - + def __init__(self, root_path): super(PackedDB, self).__init__(root_path) # list of lists with three items: @@ -44,29 +44,29 @@ def __init__(self, root_path): # self._entities = list() # lazy loaded list self._hit_count = 0 # amount of hits self._st_mtime = 0 # last modification data of our root path - + def _set_cache_(self, attr): if attr == '_entities': self._entities = list() self.update_cache(force=True) # END handle entities initialization - + def _sort_entities(self): self._entities.sort(key=lambda l: l[0], reverse=True) - + def _pack_info(self, sha): """:return: tuple(entity, index) for an item at the given sha :param sha: 20 or 40 byte sha :raise BadObject: **Note:** This method is not thread-safe, but may be hit in multi-threaded - operation. The worst thing that can happen though is a counter that + operation. The worst thing that can happen though is a counter that was not incremented, or the list being in wrong order. So we safe the time for locking here, lets see how that goes""" # presort ? if self._hit_count % self._sort_interval == 0: self._sort_entities() # END update sorting - + for item in self._entities: index = item[2](sha) if index is not None: @@ -75,14 +75,14 @@ def _pack_info(self, sha): return (item[1], index) # END index found in pack # END for each item - + # no hit, see whether we have to update packs # NOTE: considering packs don't change very often, we safe this call # and leave it to the super-caller to trigger that raise BadObject(sha) - - #{ Object DB Read - + + #{ Object DB Read + def has_object(self, sha): try: self._pack_info(sha) @@ -90,15 +90,15 @@ def has_object(self, sha): except BadObject: return False # END exception handling - + def info(self, sha): entity, index = self._pack_info(sha) return entity.info_at_index(index) - + def stream(self, sha): entity, index = self._pack_info(sha) return entity.stream_at_index(index) - + def sha_iter(self): sha_list = list() for entity in self.entities(): @@ -108,50 +108,50 @@ def sha_iter(self): yield sha_by_index(index) # END for each index # END for each entity - + def size(self): sizes = [item[1].index().size() for item in self._entities] return reduce(lambda x,y: x+y, sizes, 0) - + #} END object db read - + #{ object db write - + def store(self, istream): - """Storing individual objects is not feasible as a pack is designed to + """Storing individual objects is not feasible as a pack is designed to hold multiple objects. Writing or rewriting packs for single objects is inefficient""" raise UnsupportedOperation() - + def store_async(self, reader): # TODO: add ObjectDBRW before implementing this raise NotImplementedError() - + #} END object db write - - - #{ Interface - + + + #{ Interface + def update_cache(self, force=False): """ - Update our cache with the acutally existing packs on disk. Add new ones, + Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones - + :param force: If True, the cache will be updated even though the directory does not appear to have changed according to its modification timestamp. - :return: True if the packs have been updated so there is new information, + :return: True if the packs have been updated so there is new information, False if there was no change to the pack database""" stat = os.stat(self.root_path()) if not force and stat.st_mtime <= self._st_mtime: return False # END abort early on no change self._st_mtime = stat.st_mtime - + # packs are supposed to be prefixed with pack- by git-convention # get all pack files, figure out what changed pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) our_pack_files = set(item[1].pack().path() for item in self._entities) - + # new packs for pack_file in (pack_files - our_pack_files): # init the hit-counter/priority with the size, a good measure for hit- @@ -159,7 +159,7 @@ def update_cache(self, force=False): entity = PackEntity(pack_file) self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) # END for each new packfile - + # removed packs for pack_file in (our_pack_files - pack_files): del_index = -1 @@ -172,22 +172,22 @@ def update_cache(self, force=False): assert del_index != -1 del(self._entities[del_index]) # END for each removed pack - + # reinitialize prioritiess self._sort_entities() return True - + def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - + def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha - :param partial_binsha: binary sha with less than 20 bytes + :param partial_binsha: binary sha with less than 20 bytes :param canonical_length: length of the corresponding canonical representation. It is required as binary sha's cannot display whether the original hex sha had an odd or even number of characters - :raise AmbiguousObjectName: + :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for item in self._entities: @@ -199,11 +199,11 @@ def partial_to_complete_sha(self, partial_binsha, canonical_length): candidate = sha # END handle full sha could be found # END for each entity - + if candidate: return candidate - + # still not found ? raise BadObject(partial_binsha) - + #} END interface diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 60004a77a..368ab9a61 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -11,16 +11,16 @@ class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" - + # Configuration # Specifies the object database to use for the paths found in the alternates # file. If None, it defaults to the GitDB ObjectDBCls = None - + def __init__(self, ref_file): super(ReferenceDB, self).__init__() self._ref_file = ref_file - + def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() @@ -28,7 +28,7 @@ def _set_cache_(self, attr): else: super(ReferenceDB, self)._set_cache_(attr) # END handle attrs - + def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls if dbcls is None: @@ -36,7 +36,7 @@ def _update_dbs_from_ref_file(self): from git import GitDB dbcls = GitDB # END get db type - + # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: @@ -44,10 +44,10 @@ def _update_dbs_from_ref_file(self): except (OSError, IOError): pass # END handle alternates - + ref_paths_set = set(ref_paths) cur_ref_paths_set = set(db.root_path() for db in self._dbs) - + # remove existing for path in (cur_ref_paths_set - ref_paths_set): for i, db in enumerate(self._dbs[:]): @@ -56,7 +56,7 @@ def _update_dbs_from_ref_file(self): continue # END del matching db # END for each path to remove - + # add new # sort them to maintain order added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) @@ -72,7 +72,7 @@ def _update_dbs_from_ref_file(self): # ignore invalid paths or issues pass # END for each path to add - + def update_cache(self, force=False): # re-read alternates and update databases self._update_dbs_from_ref_file() diff --git a/gitdb/exc.py b/gitdb/exc.py index 7180fb586..47fc80912 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -7,17 +7,17 @@ class ODBError(Exception): """All errors thrown by the object database""" - + class InvalidDBRoot(ODBError): """Thrown if an object database cannot be initialized at the given path""" - + class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the + """The object with the given SHA does not exist. Instantiate with the failed sha""" - + def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) - + class ParseError(ODBError): """Thrown if the parsing of a file failed due to an invalid format""" diff --git a/gitdb/fun.py b/gitdb/fun.py index c1e73e895..9e5c44a6c 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -24,30 +24,30 @@ delta_types = (OFS_DELTA, REF_DELTA) type_id_to_type_map = { - 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", - 5 : "", # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA - } + 0 : "", # EXT 1 + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA +} type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA - ) + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA +) # used when dealing with larger streams -chunk_size = 1000*mmap.PAGESIZE +chunk_size = 1000 * mmap.PAGESIZE -__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') @@ -59,11 +59,11 @@ def _set_delta_rbound(d, size): to our size :return: d""" d.ts = size - + # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE return d - + def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static @@ -71,19 +71,19 @@ def _move_delta_lbound(d, bytes): :return: d""" if bytes == 0: return - + d.to += bytes d.so += bytes d.ts -= bytes if d.data is not None: d.data = d.data[bytes:] # END handle data - + return d - + def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) - + def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer :param bbuf: buffer providing source bytes for copy operations @@ -107,13 +107,13 @@ class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" __slots__ = ( - 'to', # start offset in the target buffer in bytes + 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None ) - + def __init__(self, to, ts, so, data): self.to = to self.ts = ts @@ -122,22 +122,22 @@ def __init__(self, to, ts, so, data): def __repr__(self): return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") - + #{ Interface - + def rbound(self): return self.to + self.ts - + def has_data(self): """:return: True if the instance has data to add to the target stream""" return self.data is not None - + #} END interface def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs. + absofs. **Note:** global method for performance only, it belongs to DeltaChunkList""" lo = 0 hi = len(dcl) @@ -153,7 +153,7 @@ def _closest_index(dcl, absofs): # END handle bound # END for each delta absofs return len(dcl)-1 - + def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed write function. @@ -166,14 +166,14 @@ def delta_list_apply(dcl, bbuf, write): # END for each dc def delta_list_slice(dcl, absofs, size, ndcl): - """:return: Subsection of this list at the given absolute offset, with the given + """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: None""" cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - lappend = ndcl.append - + lappend = ndcl.append + if cd.to != absofs: tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) _move_delta_lbound(tcd, absofs - cd.to) @@ -182,7 +182,7 @@ def delta_list_slice(dcl, absofs, size, ndcl): size -= tcd.ts cdi += 1 # END lbound overlap handling - + while cdi < slen and size: # are we larger than the current block cd = dcl[cdi] @@ -198,38 +198,38 @@ def delta_list_slice(dcl, absofs, size, ndcl): # END hadle size cdi += 1 # END for each chunk - - + + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. There are two types of lists we represent. The one was created bottom-up, working - towards the latest delta, the other kind was created top-down, working from the - latest delta down to the earliest ancestor. This attribute is queryable + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable after all processing with is_reversed.""" - + __slots__ = tuple() - + def rbound(self): """:return: rightmost extend in bytes, absolute""" if len(self) == 0: return 0 return self[-1].rbound() - + def lbound(self): """:return: leftmost byte at which this chunklist starts""" if len(self) == 0: return 0 return self[0].to - + def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - + def apply(self, bbuf, write): """Only used by public clients, internally we only use the global routines for performance""" return delta_list_apply(self, bbuf, write) - + def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate add-chunks @@ -239,7 +239,7 @@ def compress(self): return self i = 0 slen_orig = slen - + first_data_index = None while i < slen: dc = self[i] @@ -253,27 +253,27 @@ def compress(self): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data - + del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) - + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + slen = len(self) i = first_data_index + 1 - + # END concatenate data first_data_index = None continue # END skip non-data chunks - + if first_data_index is None: first_data_index = i-1 # END iterate list - + #if slen_orig != len(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self - + def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -283,39 +283,39 @@ def check_integrity(self, target_size=-1): assert self[-1].rbound() == target_size assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size # END target size verification - + if len(self) < 2: return - + # check data for dc in self: assert dc.ts > 0 if dc.has_data(): assert len(dc.data) >= dc.ts # END for each dc - + left = islice(self, 0, len(self)-1) right = iter(self) right.next() - # this is very pythonic - we might have just use index based access here, + # this is very pythonic - we might have just use index based access here, # but this could actually be faster for lft,rgt in izip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair - + class TopdownDeltaChunkList(DeltaChunkList): - """Represents a list which is generated by feeding its ancestor streams one by + """Represents a list which is generated by feeding its ancestor streams one by one""" - __slots__ = tuple() - + __slots__ = tuple() + def connect_with_next_base(self, bdcl): """Connect this chain with the next level of our base delta chunklist. The goal in this game is to mark as many of our chunks rigid, hence they - cannot be changed by any of the upcoming bases anymore. Once all our + cannot be changed by any of the upcoming bases anymore. Once all our chunks are marked like that, we can stop all processing - :param bdcl: data chunk list being one of our bases. They must be fed in + :param bdcl: data chunk list being one of our bases. They must be fed in consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams if False is returned""" @@ -326,13 +326,13 @@ def connect_with_next_base(self, bdcl): while dci < slen: dc = self[dci] dci += 1 - + # all add-chunks which are already topmost don't need additional processing if dc.data is not None: nfc += 1 continue # END skip add chunks - + # copy chunks # integrate the portion of the base list into ourselves. Lists # dont support efficient insertion ( just one at a time ), but for now @@ -341,13 +341,13 @@ def connect_with_next_base(self, bdcl): # ourselves in order to reduce the amount of insertions ... del(ccl[:]) delta_list_slice(bdcl, dc.so, dc.ts, ccl) - + # move the target bounds into place to match with our chunk ofs = dc.to - dc.so for cdc in ccl: cdc.to += ofs # END update target bounds - + if len(ccl) == 1: self[dci-1] = ccl[0] else: @@ -359,19 +359,19 @@ def connect_with_next_base(self, bdcl): del(self[dci-1:]) # include deletion of dc self.extend(ccl) self.extend(post_dci) - + slen = len(self) dci += len(ccl)-1 # deleted dc, added rest - + # END handle chunk replacement # END for each chunk - + if nfc == slen: return False # END handle completeness return True - - + + #} END structures #{ Routines @@ -386,14 +386,14 @@ def is_loose_object(m): def loose_object_header_info(m): """ - :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find("\0")].split(" ") return type_name, int(size) - + def pack_object_header_info(data): """ :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) @@ -417,7 +417,7 @@ def create_pack_object_header(obj_type, obj_size): """ :return: string defining the pack header comprised of the object type and its incompressed size in bytes - + :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte @@ -432,10 +432,10 @@ def create_pack_object_header(obj_type, obj_size): #END until size is consumed hdr += chr(c) return hdr - + def msb_size(data, offset=0): """ - :return: tuple(read_bytes, size) read the msb size from the given random + :return: tuple(read_bytes, size) read the msb size from the given random access data starting at the given byte offset""" size = 0 i = 0 @@ -452,19 +452,19 @@ def msb_size(data, offset=0): # END while in range if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size - + return i+offset, size + def loose_object_header(type, size): """ :return: string representing the loose object header, which is immediately followed by the content stream of size 'size'""" return "%s %i\0" % (type, size) - + def write_object(type, size, read, write, chunk_size=chunk_size): """ - Write the object as identified by type, size and source_stream into the + Write the object as identified by type, size and source_stream into the target_stream - + :param type: type string of the object :param size: amount of bytes to write from source_stream :param read: read method of a stream providing the content data @@ -473,26 +473,26 @@ def write_object(type, size, read, write, chunk_size=chunk_size): the routine exits, even if an error is thrown :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" tbw = 0 # total num bytes written - + # WRITE HEADER: type SP size NULL tbw += write(loose_object_header(type, size)) tbw += stream_copy(read, write, size, chunk_size) - + return tbw def stream_copy(read, write, size, chunk_size): """ - Copy a stream up to size bytes using the provided read and write methods, + Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size - + **Note:** its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written - + # WRITE ALL DATA UP TO SIZE while True: cs = min(chunk_size, size-dbw) # NOTE: not all write methods return the amount of written bytes, like - # mmap.write. Its bad, but we just deal with it ... perhaps its not + # mmap.write. Its bad, but we just deal with it ... perhaps its not # even less efficient # data_len = write(read(cs)) # dbw += data_len @@ -505,27 +505,27 @@ def stream_copy(read, write, size, chunk_size): # END check for stream end # END duplicate data return dbw - + def connect_deltas(dstreams): """ Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - + :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" tdcl = None # topmost dcl - + dcl = tdcl = TopdownDeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() delta_buf_size = ds.size - + # read header i, base_size = msb_size(db) i, target_size = msb_size(db, i) - + # interpret opcodes tbw = 0 # amount of target bytes written while i < delta_buf_size: @@ -554,15 +554,15 @@ def connect_deltas(dstreams): if (c & 0x40): cp_size |= (ord(db[i]) << 16) i += 1 - - if not cp_size: + + if not cp_size: cp_size = 0x10000 - + rbound = cp_off + cp_size if (rbound < cp_size or rbound > base_size): break - + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: @@ -575,31 +575,31 @@ def connect_deltas(dstreams): raise ValueError("unexpected delta opcode 0") # END handle command byte # END while processing delta data - + dcl.compress() - + # merge the lists ! if dsi > 0: if not tdcl.connect_with_next_base(dcl): break # END handle merge - + # prepare next base dcl = DeltaChunkList() # END for each delta stream - + return tdcl - + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file - + :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes - + **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf @@ -629,10 +629,10 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (c & 0x40): cp_size |= (ord(db[i]) << 16) i += 1 - - if not cp_size: + + if not cp_size: cp_size = 0x10000 - + rbound = cp_off + cp_size if (rbound < cp_size or rbound > src_buf_size): @@ -645,28 +645,28 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): raise ValueError("unexpected delta opcode 0") # END handle command byte # END while processing delta data - + # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" - - + + def is_equal_canonical_sha(canonical_length, match, sha1): """ :return: True if the given lhs and rhs 20 byte binary shas - The comparison will take the canonical_length of the match sha into account, + The comparison will take the canonical_length of the match sha into account, hence the comparison will only use the last 4 bytes for uneven canonical representations :param match: less than 20 byte sha :param sha1: 20 byte sha""" binary_length = canonical_length/2 if match[:binary_length] != sha1[:binary_length]: return False - + if canonical_length - binary_length and \ (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: return False # END handle uneven canonnical length return True - + #} END routines diff --git a/gitdb/pack.py b/gitdb/pack.py index 48121f026..4a0badccb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -54,9 +54,9 @@ ) from struct import ( - pack, - unpack, - ) + pack, + unpack, +) from binascii import crc32 @@ -68,10 +68,10 @@ __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') - - -#{ Utilities + + +#{ Utilities def pack_object_at(cursor, offset, as_stream): """ @@ -81,13 +81,13 @@ def pack_object_at(cursor, offset, as_stream): data to be read decompressed. :param data: random accessable data containing all required information :parma offset: offset in to the data at which the object information is located - :param as_stream: if True, a stream object will be returned that can read + :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None - + # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset @@ -111,7 +111,7 @@ def pack_object_at(cursor, offset, as_stream): # assume its a base object total_rela_offset = data_rela_offset # END handle type id - + abs_data_offset = offset + total_rela_offset if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) @@ -141,29 +141,29 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): if want_crc: crc = base_crc #END initialize crc - + while True: chunk = read(chunk_size) br += len(chunk) compressed = zstream.compress(chunk) bw += len(compressed) write(compressed) # cannot assume return value - + if want_crc: crc = crc32(compressed, crc) #END handle crc - + if len(chunk) != chunk_size: break #END copy loop - + compressed = zstream.flush() bw += len(compressed) write(compressed) if want_crc: crc = crc32(compressed, crc) #END handle crc - + return (br, bw, crc) @@ -175,26 +175,26 @@ class IndexWriter(object): in one go to the given stream **Note:** currently only writes v2 indices""" __slots__ = '_objs' - + def __init__(self): self._objs = list() - + def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - + def write(self, pack_sha, write): """Write the index file using the given write method :param pack_sha: binary sha over the whole pack that we index :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) - + sha_writer = FlexibleSha1Writer(write) sha_write = sha_writer.write sha_write(PackIndexFile.index_v2_signature) sha_write(pack(">L", PackIndexFile.index_version_default)) - + # fanout tmplist = list((0,)*256) # fanout or list with 64 bit offsets for t in self._objs: @@ -206,16 +206,16 @@ def write(self, pack_sha, write): tmplist[i+1] += v #END write each fanout entry sha_write(pack('>L', tmplist[255])) - + # sha1 ordered # save calls, that is push them into c sha_write(''.join(t[0] for t in self._objs)) - + # crc32 for t in self._objs: sha_write(pack('>L', t[1]&0xffffffff)) #END for each crc - + tmplist = list() # offset 32 for t in self._objs: @@ -226,28 +226,28 @@ def write(self, pack_sha, write): #END hande 64 bit offsets sha_write(pack('>L', ofs&0xffffffff)) #END for each offset - + # offset 64 for ofs in tmplist: sha_write(pack(">Q", ofs)) #END for each offset - + # trailer assert(len(pack_sha) == 20) sha_write(pack_sha) sha = sha_writer.sha(as_hex=False) write(sha) return sha - - + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" - + # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -258,7 +258,7 @@ class PackIndexFile(LazyMixin): def __init__(self, indexpath): super(PackIndexFile, self).__init__() self._indexpath = indexpath - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -276,91 +276,91 @@ def _set_cache_(self, attr): else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties - + # CHECK VERSION mmap = self._cursor.map() self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version - + # SETUP FUNCTIONS # setup our functions according to the actual version for fname in ('entry', 'offset', 'sha', 'crc'): setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) # END for each function to initialize - - + + # INITIALIZE DATA # byte offset is 8 if version is 2, 0 otherwise self._initialize() # END handle attributes - + #{ Access V1 - + def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + def _offset_v1(self, i): """see ``_offset_v2``""" return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - + def _sha_v1(self, i): """see ``_sha_v2``""" base = 1024 + (i*24)+4 return self._cursor.map()[base:base+20] - + def _crc_v1(self, i): """unsupported""" return 0 - + #} END access V1 - + #{ Access V2 def _entry_v2(self, i): """:return: tuple(offset, binsha, crc)""" return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - + def _offset_v2(self, i): - """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - + # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset - + return offset - + def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 return self._cursor.map()[base:base+20] - + def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + #} END access V2 - + #{ Initialization - + def _initialize(self): """initialize base data""" self._fanout_table = self._read_fanout((self._version == 2) * 8) - + if self._version == 2: self._crc_list_offset = self._sha_list_offset + self.size() * 20 self._pack_offset = self._crc_list_offset + self.size() * 4 self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base - + def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" d = self._cursor.map() @@ -370,38 +370,38 @@ def _read_fanout(self, byte_offset): append(unpack_from('>L', d, byte_offset + i*4)[0]) # END for each entry return out - + #} END initialization - + #{ Properties def version(self): return self._version - + def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - + def path(self): """:return: path to the packindexfile""" return self._indexpath - + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._cursor.map()[-40:-20] - + def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._cursor.map()[-20:] - + def offsets(self): """:return: sequence of all offsets in the order in which they were written - + **Note:** return value can be random accessed, but may be immmutable""" if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - + # networkbyteorder to something array likes more if sys.byteorder == 'little': a.byteswap() @@ -409,7 +409,7 @@ def offsets(self): else: return tuple(self.offset(index) for index in xrange(self.size())) # END handle version - + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -421,7 +421,7 @@ def sha_to_index(self, sha): if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 @@ -435,29 +435,29 @@ def sha_to_index(self, sha): # END handle midpoint # END bisect return None - + def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this index file :param partial_bin_sha: an at least two bytes of a partial binary sha - :param canonical_length: lenght of the original hexadecimal representation of the + :param canonical_length: lenght of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # fill the partial to full 20 bytes filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest + + # find lowest while lo < hi: mid = (lo + hi) / 2 c = cmp(filled_sha, get_sha(mid)) @@ -471,7 +471,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): lo = mid + 1 # END handle midpoint # END bisect - + if lo < self.size(): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): @@ -484,86 +484,86 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): # END if we have a match # END if we found something return None - + if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses def sha_to_index(self, sha): return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - + # END redefine heavy-hitter with c version + #} END properties - - + + class PackFile(LazyMixin): """A pack is a file written according to the Version 2 for git packs - + As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - - **Note:** at some point, this might be implemented using streams as well, or + + **Note:** at some point, this might be implemented using streams as well, or streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that + for some reason - one clearly doesn't want to read 10GB at once in that case""" - + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 - + # offset into our data at which the first object starts first_object_offset = 3*4 # header bytes footer_size = 20 # final sha - + def __init__(self, packpath): self._packpath = packpath - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() - + # read the header information type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - + # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" c = self._cursor content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset - + null = NullStream() while cur_offset < content_size: data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset - + stream_copy(ostream.read, null.write, ostream.size, chunk_size) cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - + + # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the + # Otherwise return the Stream object directly, its derived from the # info object if as_stream: ostream.stream.seek(0) yield ostream # END until we have read everything - + #{ Pack Information - + def size(self): - """:return: The amount of objects stored in this pack""" + """:return: The amount of objects stored in this pack""" return self._size - + def version(self): """:return: the version of this pack""" return self._version - + def data(self): """ :return: read-only data of this pack. It provides random access and usually @@ -571,22 +571,22 @@ def data(self): :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" # can use map as we are starting at offset 0. Otherwise we would have to use buffer() return self._cursor.use_region().map() - + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - + def path(self): """:return: path to the packfile""" return self._packpath #} END pack information - + #{ Pack Specific - + def collect_streams(self, offset): """ :return: list of pack streams which are required to build the object - at the given offset. The first entry of the list is the object at offset, + at the given offset. The first entry of the list is the object at offset, the last one is either a full object, or a REF_Delta stream. The latter type needs its reference object to be locked up in an ODB to form a valid delta chain. @@ -601,7 +601,7 @@ def collect_streams(self, offset): offset = ostream.pack_offset - ostream.delta_info else: # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter + # else is either an object, or a ref delta, in the latter # case someone else has to find it break # END handle type @@ -609,55 +609,55 @@ def collect_streams(self, offset): return out #} END pack specific - + #{ Read-Database like Interface - + def info(self, offset): """Retrieve information about the object at the given file-absolute offset - + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - + def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information - + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - + def stream_iter(self, start_offset=0): """ - :return: iterator yielding OPackStream compatible instances, allowing + :return: iterator yielding OPackStream compatible instances, allowing to access the data in the pack directly. - :param start_offset: offset to the first object to iterate. If 0, iteration + :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. - + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) - + #} END Read-Database like Interface - - + + class PackEntity(LazyMixin): - """Combines the PackIndexFile and the PackFile into one, allowing the + """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file + + __slots__ = ( '_index', # our index file '_pack', # our pack file '_offset_map' # on demand dict mapping one offset to the next consecutive one ) - + IndexFileCls = PackIndexFile PackFileCls = PackFile - + def __init__(self, pack_or_index_path): """Initialize ourselves with the path to the respective pack or index file""" basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - + def _set_cache_(self, attr): # currently this can only be _offset_map # TODO: make this a simple sorted offset array which can be bisected @@ -666,7 +666,7 @@ def _set_cache_(self, attr): offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" - + offset_map = None if len(offsets_sorted) == 1: offset_map = { offsets_sorted[0] : last_offset } @@ -675,21 +675,21 @@ def _set_cache_(self, attr): iter_offsets_plus_one = iter(offsets_sorted) iter_offsets_plus_one.next() consecutive = izip(iter_offsets, iter_offsets_plus_one) - + offset_map = dict(consecutive) - + # the last offset is not yet set offset_map[offsets_sorted[-1]] = last_offset # END handle offset amount self._offset_map = offset_map - + def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) if index is None: raise BadObject(sha) return index - + def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha @@ -697,7 +697,7 @@ def _iter_objects(self, as_stream): for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index - + def _object(self, sha, as_stream, index=-1): """:return: OInfo or OStream object providing information about the given sha :param index: if not -1, its assumed to be the sha's index in the IndexFile""" @@ -714,86 +714,86 @@ def _object(self, sha, as_stream, index=-1): packstream = self._pack.stream(offset) return OStream(sha, packstream.type, packstream.size, packstream.stream) # END handle non-deltas - + # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, + # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) + + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) # END handle non-deltas - + # deltas are a little tougher - unpack the first bytes to obtain # the actual target size, as opposed to the size of the delta data streams = self.collect_streams_at_offset(offset) buf = streams[0].read(512) offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - + # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) + return OInfo(sha, streams[-1].type, target_size) # END handle stream - + #{ Read-Database like Interface - + def info(self, sha): """Retrieve information about the object identified by the given sha - + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" return self._object(sha, False) - + def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha - + :param sha: 20 byte sha1 - :raise BadObject: + :raise BadObject: :return: OStream instance, with 20 byte sha""" return self._object(sha, True) def info_at_index(self, index): """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, False, index) - + def stream_at_index(self, index): - """As ``stream``, but uses a PackIndexFile compatible index to refer to the + """As ``stream``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, True, index) - + #} END Read-Database like Interface - - #{ Interface + + #{ Interface def pack(self): """:return: the underlying pack file instance""" return self._pack - + def index(self): """:return: the underlying pack index file instance""" return self._index - + def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - - :param use_crc: if True, the index' crc is run over the compressed stream of + + :param use_crc: if True, the index' crc is run over the compressed stream of the object, which is much faster than checking the sha1. It is also more prone to unnoticed corruption or manipulation. :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is - a delta, this only verifies that the delta's data is valid, not the - data of the actual undeltified object, as it depends on more than + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha - + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -801,12 +801,12 @@ def is_valid_stream(self, sha, use_crc=False): if self._index.version() < 2: raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") # END handle index version - + index = self._sha_to_index(sha) offset = self._index.offset(index) next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - + # create the current crc value, on the compressed object data # Read it in chunks, without copying the data crc_update = zlib.crc32 @@ -819,7 +819,7 @@ def is_valid_stream(self, sha, use_crc=False): this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - + # crc returns signed 32 bit numbers, the AND op forces it into unsigned # mode ... wow, sneaky, from dulwich. return (this_crc_value & 0xffffffff) == crc_value @@ -828,7 +828,7 @@ def is_valid_stream(self, sha, use_crc=False): stream = self._object(sha, as_stream=True) # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) - + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification @@ -839,21 +839,21 @@ def info_iter(self): :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) - + def stream_iter(self): """ :return: iterator over all objects in this pack. The iterator yields OStream instances""" return self._iter_objects(as_stream=True) - + def collect_streams_at_offset(self, offset): """ As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` - + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) - + # try to resolve the last one if needed. It is assumed to be either # a REF delta, or a base object, as OFFSET deltas are resolved by the pack if streams[-1].type_id == REF_DELTA: @@ -866,54 +866,54 @@ def collect_streams_at_offset(self, offset): stream = self._pack.stream(self._index.offset(sindex)) streams.append(stream) else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who # would do that ;) ? We can handle it though stream = self._pack.stream(stream.delta_info) streams.append(stream) # END handle ref delta # END resolve ref streams # END resolve streams - + return streams - + def collect_streams(self, sha): """ As ``PackFile.collect_streams``, but takes a sha instead of an offset. Additionally, ref_delta streams will be resolved within this pack. If this is not possible, the stream will be left alone, hence it is adivsed - to check for unresolved ref-deltas and resolve them before attempting to + to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. - + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect - :return: list of streams, first being the actual object delta, the last being + :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. :raise BadObject:""" return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - + + @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, + def write_pack(cls, object_iter, pack_write, index_write=None, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. The respective index is produced as well if index_write is not Non. - + :param object_iter: iterator yielding odb output objects :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding to the pack. - :param object_count: if you can provide the amount of objects in your iteration, - this would be the place to put it. Otherwise we have to pre-iterate and store + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store all items into a list to get the number, which uses more memory than necessary. :param zlib_compression: the zlib compression level to use :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack and over all contents of the index. If index_write was None, index_binsha will be None - + **Note:** The destination of the write functions is up to the user. It could be a socket, or a file for instance - + **Note:** writes only undeltified objects""" objs = object_iter if not object_count: @@ -922,26 +922,26 @@ def write_pack(cls, object_iter, pack_write, index_write=None, #END handle list type object_count = len(objs) #END handle object - + pack_writer = FlexibleSha1Writer(pack_write) pwrite = pack_writer.write ofs = 0 # current offset into the pack file index = None wants_index = index_write is not None - + # write header pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) ofs += 12 - + if wants_index: index = IndexWriter() #END handle index header - + actual_count = 0 for obj in objs: actual_count += 1 crc = 0 - + # object header hdr = create_pack_object_header(obj.type_id, obj.size) if index_write: @@ -950,7 +950,7 @@ def write_pack(cls, object_iter, pack_write, index_write=None, crc = None #END handle crc pwrite(hdr) - + # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream @@ -959,54 +959,54 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if wants_index: index.append(obj.binsha, crc, ofs) #END handle index - + ofs += len(hdr) + bw if actual_count == object_count: break #END abort once we are done #END for each object - + if actual_count != object_count: raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) #END count assertion - + # write footer pack_sha = pack_writer.sha(as_hex = False) assert len(pack_sha) == 20 pack_write(pack_sha) ofs += len(pack_sha) # just for completeness ;) - + index_sha = None if wants_index: index_sha = index.write(pack_sha, index_write) #END handle index - + return pack_sha, index_sha - + @classmethod def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """Create a new on-disk entity comprised of a properly named pack file and a properly named and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files :return: PackEntity instance initialized with the new pack - + **Note:** for more information on the other parameters see the write_pack method""" pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) index_write = lambda d: os.write(index_fd, d) - + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) os.close(pack_fd) os.close(index_fd) - + fmt = "pack-%s.%s" new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) os.rename(pack_path, new_pack_path) os.rename(index_path, new_index_path) - + return cls(new_pack_path) - - + + #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 6441b1e1a..b21c39c9d 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,7 @@ except ImportError: pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', 'FDStream', 'NullStream') @@ -41,27 +41,27 @@ #{ RO Streams class DecompressMemMapReader(LazyMixin): - """Reads data in chunks from a memory map and decompresses it. The client sees + """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly - - A constraint on the total size of bytes is activated, simulating + + A constraint on the total size of bytes is activated, simulating a logical file within a possibly larger physical memory area - - To read efficiently, you clearly don't want to read individual bytes, instead, + + To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. - - **Note:** The chunk-size should be carefully selected as it will involve quite a bit - of string copying due to the way the zlib is implemented. Its very wasteful, - hence we try to find a good tradeoff between allocation time and number of + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', '_cbr', '_phi') - + max_read_size = 512*1024 # currently unused - + def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no size""" @@ -77,22 +77,22 @@ def __init__(self, m, close_on_deletion, size=None): self._cbr = 0 # number of compressed bytes read self._phi = False # is True if we parsed the header info self._close = close_on_deletion # close the memmap on deletion ? - + def _set_cache_(self, attr): assert attr == '_s' - # only happens for size, which is a marker to indicate we still + # only happens for size, which is a marker to indicate we still # have to parse the header from the stream self._parse_header_info() - + def __del__(self): if self._close: self._m.close() # END handle resource freeing - + def _parse_header_info(self): - """If this stream contains object data, parse the header info and skip the + """If this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content - + :return: parsed type_string, size""" # read header maxb = 512 # should really be enough, cgit uses 8192 I believe @@ -102,28 +102,28 @@ def _parse_header_info(self): type, size = hdr[:hdrend].split(" ") size = int(size) self._s = size - + # adjust internal state to match actual header length that we ignore # The buffer will be depleted first on future reads self._br = 0 hdrend += 1 # count terminating \0 self._buf = StringIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend - + self._phi = True - + return type, size - - #{ Interface - + + #{ Interface + @classmethod def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream - This method parses the object header from m and returns the parsed + This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. - + :param m: memory map on which to oparate. It must be object data ( header + contents ) - :param close_on_deletion: if True, the memory map will be closed once we are + :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) type, size = inst._parse_header_info() @@ -131,30 +131,30 @@ def new(self, m, close_on_deletion=False): def data(self): """:return: random access compatible data we are working on""" - return self._m - + return self._m + def compressed_bytes_read(self): """ - :return: number of compressed bytes read. This includes the bytes it + :return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )""" # ABSTRACT: When decompressing a byte stream, it can be that the first - # x bytes which were requested match the first x bytes in the loosely + # x bytes which were requested match the first x bytes in the loosely # compressed datastream. This is the worst-case assumption that the reader # does, it assumes that it will get at least X bytes from X compressed bytes # in call cases. - # The caveat is that the object, according to our known uncompressed size, + # The caveat is that the object, according to our known uncompressed size, # is already complete, but there are still some bytes left in the compressed # stream that contribute to the amount of compressed bytes. # How can we know that we are truly done, and have read all bytes we need - # to read ? - # Without help, we cannot know, as we need to obtain the status of the + # to read ? + # Without help, we cannot know, as we need to obtain the status of the # decompression. If it is not finished, we need to decompress more data # until it is finished, to yield the actual number of compressed bytes # belonging to the decompressed object - # We are using a custom zlib module for this, if its not present, + # We are using a custom zlib module for this, if its not present, # we try to put in additional bytes up for decompression if feasible # and check for the unused_data. - + # Only scrub the stream forward if we are officially done with the # bytes we were to have. if self._br == self._s and not self._zip.unused_data: @@ -171,45 +171,45 @@ def compressed_bytes_read(self): self.read(mmap.PAGESIZE) # END scrub-loop default zlib # END handle stream scrubbing - + # reset bytes read, just to be sure self._br = self._s # END handle stream scrubbing - + # unused data ends up in the unconsumed tail, which was removed # from the count already return self._cbr - - #} END interface - + + #} END interface + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset - + self._zip = zlib.decompressobj() self._br = self._cws = self._cwe = self._cbr = 0 if self._phi: self._phi = False del(self._s) # trigger header parsing on first access # END skip header - + def read(self, size=-1): if size < 1: size = self._s - self._br else: size = min(size, self._s - self._br) # END clamp size - + if size == 0: return str() # END handle depletion - - - # deplete the buffer, then just continue using the decompress object - # which has an own buffer. We just need this to transparently parse the + + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the # header from the zlib stream dat = str() if self._buf: @@ -223,26 +223,26 @@ def read(self, size=-1): dat = self._buf.read() # ouch, duplicates data size -= self._buflen self._br += self._buflen - + self._buflen = 0 self._buf = None # END handle buffer len # END handle buffer - + # decompress some data - # Abstract: zlib needs to operate on chunks of our memory map ( which may + # Abstract: zlib needs to operate on chunks of our memory map ( which may # be large ), as it will otherwise and always fill in the 'unconsumed_tail' - # attribute which possible reads our whole map to the end, forcing + # attribute which possible reads our whole map to the end, forcing # everything to be read from disk even though just a portion was requested. - # As this would be a nogo, we workaround it by passing only chunks of data, - # moving the window into the memory map along as we decompress, which keeps + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps # the tail smaller than our chunk-size. This causes 'only' the chunk to be # copied once, and another copy of a part of it when it creates the unconsumed # tail. We have to use it to hand in the appropriate amount of bytes durin g # the next read. tail = self._zip.unconsumed_tail if tail: - # move the window, make it as large as size demands. For code-clarity, + # move the window, make it as large as size demands. For code-clarity, # we just take the chunk from our map again instead of reusing the unconsumed # tail. The latter one would safe some memory copying, but we could end up # with not getting enough data uncompressed, so we had to sort that out as well. @@ -253,18 +253,18 @@ def read(self, size=-1): else: cws = self._cws self._cws = self._cwe - self._cwe = cws + size + self._cwe = cws + size # END handle tail - - + + # if window is too small, make it larger so zip can decompress something if self._cwe - self._cws < 8: self._cwe = self._cws + 8 # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... + + # takes a slice, but doesn't copy the data, it says ... indata = buffer(self._m, self._cws, self._cwe - self._cws) - + # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) dcompdat = self._zip.decompress(indata, size) @@ -274,13 +274,13 @@ def read(self, size=-1): # if we hit the end of the stream self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) - + if dat: dcompdat = dat + dcompdat # END prepend our cached data - - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. + + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. # Recursively resolve that. # Note: dcompdat can be empty even though we still appear to have bytes # to read, if we are called by compressed_bytes_read - it manipulates @@ -290,30 +290,30 @@ def read(self, size=-1): # END handle special case return dcompdat - + class DeltaApplyReader(LazyMixin): - """A reader which dynamically applies pack deltas to a base object, keeping the + """A reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. - - The size of the final object is only obtainable once all deltas have been + + The size of the final object is only obtainable once all deltas have been applied, unless it is retrieved from a pack index. - + The uncompressed Delta has the following layout (MSB being a most significant bit encoded dynamic size): - + * MSB Source Size - the size of the base against which the delta was created * MSB Target Size - the size of the resulting data after the delta was applied * A list of one byte commands (cmd) which are followed by a specific protocol: - + * cmd & 0x80 - copy delta_data[offset:offset+size] - + * Followed by an encoded offset into the delta data * Followed by an encoded size of the chunk to copy - + * cmd & 0x7f - insert - + * insert cmd bytes from the delta buffer into the output stream - + * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( @@ -321,38 +321,38 @@ class DeltaApplyReader(LazyMixin): "_dstreams", # tuple of delta stream readers "_mm_target", # memory map of the delta-applied data "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read + "_br" # number of bytes read ) - + #{ Configuration k_max_memory_move = 250*1000*1000 #} END configuration - + def __init__(self, stream_list): - """Initialize this instance with a list of streams, the first stream being + """Initialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the base object onto which to apply the deltas""" assert len(stream_list) > 1, "Need at least one delta and one base stream" - + self._bstream = stream_list[-1] self._dstreams = tuple(stream_list[:-1]) self._br = 0 - + def _set_cache_too_slow_without_c(self, attr): - # the direct algorithm is fastest and most direct if there is only one + # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python, every function call costs + # than X - definitely the case in python, every function call costs # huge amounts of time # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) - - # Aggregate all deltas into one delta in reverse order. Hence we take + + # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) - + # call len directly, as the (optional) c version doesn't implement the sequence # protocol if dcl.rbound() == 0: @@ -360,22 +360,22 @@ def _set_cache_too_slow_without_c(self, attr): self._mm_target = allocate_memory(0) return # END handle empty list - + self._size = dcl.rbound() self._mm_target = allocate_memory(self._size) - + bbuf = allocate_memory(self._bstream.size) stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - + # APPLY CHUNKS write = self._mm_target.write dcl.apply(bbuf, write) - + self._mm_target.seek(0) - + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" - + # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the # delta is not peaked into, causing less overhead. @@ -388,37 +388,37 @@ def _set_cache_brute_(self, attr): buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream - + # sanity check - the first delta to apply should have the same source # size as our actual base stream base_size = self._bstream.size target_size = max_target_size - + # if we have more than 1 delta to apply, we will swap buffers, hence we must # assure that all buffers we use are large enough to hold all the results if len(self._dstreams) > 1: base_size = target_size = max(base_size, max_target_size) # END adjust buffer sizes - - + + # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) - + # allocate memory map large enough for the largest (intermediate) target - # We will use it as scratch space for all delta ops. If the final + # We will use it as scratch space for all delta ops. If the final # target buffer is smaller than our allocated space, we just use parts # of it upon return. tbuf = allocate_memory(target_size) - - # for each delta to apply, memory map the decompressed delta and + + # for each delta to apply, memory map the decompressed delta and # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. final_target_size = None for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): - # allocate a buffer to hold all delta data - fill in the data for + # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) # The dbuf buffer contains commands after the first two MSB sizes, the @@ -427,37 +427,37 @@ def _set_cache_brute_(self, attr): ddata.write(dbuf) # read the rest from the stream. The size we give is larger than necessary stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - + ####################################################################### if 'c_apply_delta' in globals(): c_apply_delta(bbuf, ddata, tbuf); else: apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### - - # finally, swap out source and target buffers. The target is now the + + # finally, swap out source and target buffers. The target is now the # base for the next delta to apply bbuf, tbuf = tbuf, bbuf bbuf.seek(0) tbuf.seek(0) final_target_size = target_size # END for each delta to apply - + # its already seeked to 0, constrain it to the actual size # NOTE: in the end of the loop, it swaps buffers, hence our target buffer # is not tbuf, but bbuf ! self._mm_target = bbuf self._size = final_target_size - - + + #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ else: _set_cache_ = _set_cache_too_slow_without_c - + #} END configuration - + def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: @@ -467,63 +467,63 @@ def read(self, count=0): data = self._mm_target.read(count) self._br += len(data) return data - + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading - + :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self._br = 0 self._mm_target.seek(0) - - #{ Interface - + + #{ Interface + @classmethod def new(cls, stream_list): """ Convert the given list of streams into a stream which resolves deltas when reading from it. - + :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. - - :return: Non-Delta OPackStream object whose stream can be used to obtain + + :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handled""" if len(stream_list) < 2: raise ValueError("Need at least two streams") # END single object special handling - + if stream_list[-1].type_id in delta_types: raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream - + return cls(stream_list) - + #} END interface - - + + #{ OInfo like Interface - + @property def type(self): return self._bstream.type - + @property def type_id(self): return self._bstream.type_id - + @property def size(self): """:return: number of uncompressed bytes in the stream""" return self._size - - #} END oinfo like interface - - + + #} END oinfo like interface + + #} END RO streams @@ -533,7 +533,7 @@ class Sha1Writer(object): """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" __slots__ = "sha1" - + def __init__(self): self.sha1 = make_sha() @@ -545,29 +545,29 @@ def write(self, data): self.sha1.update(data) return len(data) - # END stream interface + # END stream interface #{ Interface - + def sha(self, as_hex = False): """:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise""" if as_hex: return self.sha1.hexdigest() return self.sha1.digest() - - #} END interface + + #} END interface class FlexibleSha1Writer(Sha1Writer): - """Writer producing a sha1 while passing on the written bytes to the given + """Writer producing a sha1 while passing on the written bytes to the given write function""" __slots__ = 'writer' - + def __init__(self, writer): Sha1Writer.__init__(self) self.writer = writer - + def write(self, data): Sha1Writer.write(self, data) self.writer(data) @@ -580,18 +580,18 @@ def __init__(self): Sha1Writer.__init__(self) self.buf = StringIO() self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - + def __getattr__(self, attr): return getattr(self.buf, attr) - + def write(self, data): alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) return alen - + def close(self): self.buf.write(self.zip.flush()) - + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" @@ -599,23 +599,23 @@ def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): raise ValueError("Can only seek to position 0") # END handle offset self.buf.seek(0) - + def getvalue(self): """:return: string value from the current stream position to the end""" return self.buf.getvalue() class FDCompressedSha1Writer(Sha1Writer): - """Digests data written to it, making the sha available, then compress the + """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor - + **Note:** operates on raw file descriptors **Note:** for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") - + # default exception exc = IOError("Failed to write all bytes to filedescriptor") - + def __init__(self, fd): super(FDCompressedSha1Writer, self).__init__() self.fd = fd @@ -643,33 +643,33 @@ def close(self): class FDStream(object): - """A simple wrapper providing the most basic functions on a file descriptor + """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership""" __slots__ = ("_fd", '_pos') def __init__(self, fd): self._fd = fd self._pos = 0 - + def write(self, data): self._pos += len(data) os.write(self._fd, data) - + def read(self, count=0): if count == 0: count = os.path.getsize(self._filepath) # END handle read everything - + bytes = os.read(self._fd, count) self._pos += len(bytes) return bytes - + def fileno(self): return self._fd - + def tell(self): return self._pos - + def close(self): close(self._fd) @@ -678,17 +678,15 @@ class NullStream(object): """A stream that does nothing but providing a stream interface. Use it like /dev/null""" __slots__ = tuple() - + def read(self, size=0): return '' - + def close(self): pass - + def write(self, data): return len(data) #} END W streams - - diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index f8059447f..e84e503b6 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -5,7 +5,7 @@ import gitdb.util -#{ Initialization +#{ Initialization def _init_pool(): """Assure the pool is actually threaded""" size = 2 diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 62614ee5c..dc89039dd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -9,16 +9,16 @@ ZippedStoreShaWriter, fixture_path, TestBase - ) +) from gitdb.stream import Sha1Writer from gitdb.base import ( - IStream, - OStream, - OInfo - ) - + IStream, + OStream, + OInfo +) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type @@ -28,14 +28,14 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') - + class TestDBBase(TestBase): """Base class providing testing routines on databases""" - + # data two_lines = "1234\nhello world" all_data = (two_lines, ) - + def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() @@ -46,23 +46,23 @@ def _assert_object_writing_simple(self, db): new_istream = db.store(istream) assert new_istream is istream assert db.has_object(istream.binsha) - + info = db.info(istream.binsha) assert isinstance(info, OInfo) assert info.type == istream.type and info.size == istream.size - + stream = db.stream(istream.binsha) assert isinstance(stream, OStream) assert stream.binsha == info.binsha and stream.type == info.type assert stream.read() == data # END for each item - + assert db.size() == null_objs + ni shas = list(db.sha_iter()) assert len(shas) == db.size() assert len(shas[0]) == 20 - - + + def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW **Note:** requires write access to the database""" @@ -76,25 +76,25 @@ def _assert_object_writing(self, db): ostream = ostreamcls() assert isinstance(ostream, Sha1Writer) # END create ostream - + prev_ostream = db.set_ostream(ostream) - assert type(prev_ostream) in ostreams or prev_ostream in ostreams - + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + istream = IStream(str_blob_type, len(data), StringIO(data)) - + # store returns same istream instance, with new sha set my_istream = db.store(istream) sha = istream.binsha assert my_istream is istream assert db.has_object(sha) != dry_run - assert len(sha) == 20 - + assert len(sha) == 20 + # verify data - the slow way, we want to run code if not dry_run: info = db.info(sha) assert str_blob_type == info.type assert info.size == len(data) - + ostream = db.stream(sha) assert ostream.read() == data assert ostream.type == str_blob_type @@ -102,29 +102,29 @@ def _assert_object_writing(self, db): else: self.failUnlessRaises(BadObject, db.info, sha) self.failUnlessRaises(BadObject, db.stream, sha) - + # DIRECT STREAM COPY # our data hase been written in object format to the StringIO # we pasesd as output stream. No physical database representation # was created. - # Test direct stream copy of object streams, the result must be + # Test direct stream copy of object streams, the result must be # identical to what we fed in ostream.seek(0) istream.stream = ostream assert istream.binsha is not None prev_sha = istream.binsha - + db.set_ostream(ZippedStoreShaWriter()) db.store(istream) assert istream.binsha == prev_sha new_ostream = db.ostream() - + # note: only works as long our store write uses the same compression # level, which is zip_best assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode - + def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 @@ -134,23 +134,23 @@ def istream_generator(offset=0, ni=ni): yield IStream(str_blob_type, len(data), StringIO(data)) # END for each item # END generator utility - + # for now, we are very trusty here as we expect it to work if it worked # in the single-stream case - + # write objects reader = IteratorReader(istream_generator()) istream_reader = db.store_async(reader) istreams = istream_reader.read() # read all assert istream_reader.task().error() is None assert len(istreams) == ni - + for stream in istreams: assert stream.error is None assert len(stream.binsha) == 20 assert isinstance(stream, IStream) # END assert each stream - + # test has-object-async - we must have all previously added ones reader = IteratorReader( istream.binsha for istream in istreams ) hasobject_reader = db.has_object_async(reader) @@ -160,11 +160,11 @@ def istream_generator(offset=0, ni=ni): count += 1 # END for each sha assert count == ni - + # read the objects we have just written reader = IteratorReader( istream.binsha for istream in istreams ) ostream_reader = db.stream_async(reader) - + # read items individually to prevent hitting possible sys-limits count = 0 for ostream in ostream_reader: @@ -173,30 +173,30 @@ def istream_generator(offset=0, ni=ni): # END for each ostream assert ostream_reader.task().error() is None assert count == ni - + # get info about our items reader = IteratorReader( istream.binsha for istream in istreams ) info_reader = db.info_async(reader) - + count = 0 for oinfo in info_reader: assert isinstance(oinfo, OInfo) count += 1 # END for each oinfo instance assert count == ni - - + + # combined read-write using a converter # add 2500 items, and obtain their output streams nni = 2500 reader = IteratorReader(istream_generator(offset=ni, ni=nni)) istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - + istream_reader = db.store_async(reader) istream_reader.set_post_cb(istream_to_sha) - + ostream_reader = db.stream_async(istream_reader) - + count = 0 # read it individually, otherwise we might run into the ulimit for ostream in ostream_reader: @@ -204,5 +204,3 @@ def istream_generator(offset=0, ni=ni): count += 1 # END for each ostream assert count == nni - - diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 1ef577aa3..4894c6a79 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,15 +7,15 @@ from gitdb.db import GitDB from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex - + class TestGitDB(TestDBBase): - + def test_reading(self): gdb = GitDB(fixture_path('../../../.git/objects')) - + # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 - + # access should be possible gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) @@ -23,25 +23,25 @@ def test_reading(self): assert gdb.size() > 200 sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() - - - # This is actually a test for compound functionality, but it doesn't + + + # This is actually a test for compound functionality, but it doesn't # have a separate test module # test partial shas # this one as uneven and quite short assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") - + # mix even/uneven hexshas for i, binsha in enumerate(sha_list): assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha # END for each sha - + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") - + @with_rw_directory def test_writing(self, path): gdb = GitDB(path) - + # its possible to write objects self._assert_object_writing(gdb) self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index d7e1d01b0..e295db563 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -6,29 +6,28 @@ from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex - + class TestLooseDB(TestDBBase): - + @with_rw_directory def test_basics(self, path): ldb = LooseObjectDB(path) - + # write data self._assert_object_writing(ldb) self._assert_object_writing_async(ldb) - + # verify sha iteration and size shas = list(ldb.sha_iter()) assert shas and len(shas[0]) == 20 - + assert len(shas) == ldb.size() - + # verify find short object long_sha = bin_to_hex(shas[-1]) for short_sha in (long_sha[:20], long_sha[:5]): assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha # END for each sha - + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') # raises if no object could be foudn - diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index df428e2b7..ac9bc34e9 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -4,27 +4,27 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( - MemoryDB, - LooseObjectDB - ) - + MemoryDB, + LooseObjectDB +) + class TestMemoryDB(TestDBBase): - + @with_rw_directory def test_writing(self, path): mdb = MemoryDB() - + # write data self._assert_object_writing_simple(mdb) - + # test stream copy ldb = LooseObjectDB(path) assert ldb.size() == 0 num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) assert num_streams_copied == mdb.size() - + assert ldb.size() == mdb.size() for sha in mdb.sha_iter(): assert ldb.has_object(sha) - assert ldb.stream(sha).read() == mdb.stream(sha).read() + assert ldb.stream(sha).read() == mdb.stream(sha).read() # END verify objects where copied and are equal diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f4cb5bbc6..0d9110a01 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -12,45 +12,45 @@ import random class TestPackDB(TestDBBase): - + @with_rw_directory @with_packs_rw def test_writing(self, path): pdb = PackedDB(path) - + # on demand, we init our pack cache num_packs = len(pdb.entities()) assert pdb._st_mtime != 0 - - # test pack directory changed: + + # test pack directory changed: # packs removed - rename a file, should affect the glob pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" os.rename(pack_path, new_pack_path) - + pdb.update_cache(force=True) assert len(pdb.entities()) == num_packs - 1 - + # packs added os.rename(new_pack_path, pack_path) pdb.update_cache(force=True) assert len(pdb.entities()) == num_packs - + # bang on the cache # access the Entities directly, as there is no iteration interface # yet ( or required for now ) sha_list = list(pdb.sha_iter()) assert len(sha_list) == pdb.size() - + # hit all packs in random order random.shuffle(sha_list) - + for sha in sha_list: info = pdb.info(sha) stream = pdb.stream(sha) # END for each sha to query - - + + # test short finding - be a bit more brutal here max_bytes = 19 min_bytes = 2 @@ -64,10 +64,10 @@ def test_writing(self, path): pass # valid, we can have short objects # END exception handling # END for each sha to find - + # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... + # but in our pack, there is no ambigious ... # assert num_ambiguous - + # non-existing self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 1637bff74..086446823 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -6,14 +6,14 @@ from gitdb.db import ReferenceDB from gitdb.util import ( - NULL_BIN_SHA, - hex_to_bin - ) + NULL_BIN_SHA, + hex_to_bin +) import os - + class TestReferenceDB(TestDBBase): - + def make_alt_file(self, alt_path, alt_list): """Create an alternates file which contains the given alternates. The list can be empty""" @@ -21,40 +21,38 @@ def make_alt_file(self, alt_path, alt_list): for alt in alt_list: alt_file.write(alt + "\n") alt_file.close() - + @with_rw_directory def test_writing(self, path): NULL_BIN_SHA = '\0' * 20 - + alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) assert len(rdb.databases()) == 0 assert rdb.size() == 0 assert len(list(rdb.sha_iter())) == 0 - + # try empty, non-existing assert not rdb.has_object(NULL_BIN_SHA) - - + + # setup alternate file # add two, one is invalid own_repo_path = fixture_path('../../../.git/objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 - + # we should now find a default revision of ours gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert rdb.has_object(gitdb_sha) - + # remove valid self.make_alt_file(alt_path, ["just/one/invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 0 - + # add valid self.make_alt_file(alt_path, [own_repo_path]) rdb.update_cache() assert len(rdb.databases()) == 1 - - diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ac8473a4e..685af2fad 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,12 +4,12 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( - OStream, + OStream, ) -from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter - ) +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) from gitdb.util import zlib @@ -30,14 +30,14 @@ class TestBase(unittest.TestCase): """Base class for all tests""" - + #} END bases #{ Decorators def with_rw_directory(func): - """Create a temporary directory which can be written to, remove it if the + """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) @@ -52,7 +52,7 @@ def wrapper(self): raise finally: # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as + # a windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. if not keep: @@ -60,20 +60,20 @@ def wrapper(self): shutil.rmtree(path) # END handle exception # END wrapper - + wrapper.__name__ = func.__name__ return wrapper def with_packs_rw(func): - """Function that provides a path into which the packs for testing should be + """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" def wrapper(self, path): src_pack_glob = fixture_path('packs/*') copy_files_globbed(src_pack_glob, path, hard_link_ok=True) return func(self, path) # END wrapper - + wrapper.__name__ = func.__name__ return wrapper @@ -86,10 +86,10 @@ def fixture_path(relapath=''): :param relapath: relative path into the fixtures directory, or '' to obtain the fixture directory itself""" return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) - + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): """Copy all files found according to the given source glob into the target directory - :param hard_link_ok: if True, hard links will be created if possible. Otherwise + :param hard_link_ok: if True, hard links will be created if possible. Otherwise the files will be copied""" for src_file in glob.glob(source_glob): if hard_link_ok and hasattr(os, 'link'): @@ -103,7 +103,7 @@ def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): shutil.copy(src_file, target_dir) # END try hard link # END for each file to copy - + def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes @@ -121,7 +121,7 @@ def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata + data - + def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" @@ -137,14 +137,14 @@ def __init__(self): self.was_read = False self.bytes = 0 self.closed = False - + def read(self, size): self.was_read = True self.bytes = size - + def close(self): self.closed = True - + def _assert(self): assert self.was_read @@ -153,10 +153,9 @@ class DeriveTest(OStream): def __init__(self, sha, type, size, stream, *args, **kwargs): self.myarg = kwargs.pop('myarg') self.args = args - + def _assert(self): assert self.args assert self.myarg #} END stream utilitiess - diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index d4ce428c3..76b4d2709 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -6,7 +6,7 @@ from lib import ( TestBase, DummyStream, - DeriveTest, + DeriveTest, ) from gitdb import * @@ -20,33 +20,33 @@ class TestBaseTypes(TestBase): - + def test_streams(self): # test info sha = NULL_BIN_SHA s = 20 blob_id = 3 - + info = OInfo(sha, str_blob_type, s) assert info.binsha == sha assert info.type == str_blob_type assert info.type_id == blob_id assert info.size == s - + # test pack info # provides type_id pinfo = OPackInfo(0, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - - + + # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) @@ -56,33 +56,33 @@ def test_streams(self): assert stream.bytes == 15 ostream.read(20) assert stream.bytes == 20 - + # test packstream postream = OPackStream(*(pinfo + (stream, ))) assert postream.stream is stream postream.read(10) stream._assert() assert stream.bytes == 10 - + # test deltapackstream dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) dpostream.stream is stream dpostream.read(5) stream._assert() assert stream.bytes == 5 - + # derive with own args DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - + # test istream istream = IStream(str_blob_type, s, stream) assert istream.binsha == None istream.binsha = sha assert istream.binsha == sha - + assert len(istream.binsha) == 20 assert len(istream.hexsha) == 40 - + assert istream.size == s istream.size = s * 2 istream.size == s * 2 @@ -92,7 +92,7 @@ def test_streams(self): assert istream.stream is stream istream.stream = None assert istream.stream is None - + assert istream.error is None istream.error = Exception() assert isinstance(istream.error, Exception) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 611ae4299..f45063b1e 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -11,17 +11,17 @@ from cStringIO import StringIO from async import IteratorReader - + class TestExamples(TestBase): - + def test_base(self): ldb = LooseObjectDB(fixture_path("../../../.git/objects")) - + for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) ostream = ldb.stream(sha1) assert oinfo[:3] == ostream[:3] - + assert len(ostream.read()) == ostream.size assert ldb.has_object(oinfo.binsha) # END for each sha in database @@ -32,33 +32,33 @@ def test_base(self): except UnboundLocalError: pass # END ignore exception if there are no loose objects - + data = "my data" istream = IStream("blob", len(data), StringIO(data)) - + # the object does not yet have a sha assert istream.binsha is None ldb.store(istream) # now the sha is set assert len(istream.binsha) == 20 assert ldb.has_object(istream.binsha) - - + + # async operation # Create a reader from an iterator reader = IteratorReader(ldb.sha_iter()) - + # get reader for object streams info_reader = ldb.stream_async(reader) - + # read one info = info_reader.read(1)[0] - + # read all the rest until depletion ostreams = info_reader.read() - + # set the pool to use two threads pool.set_size(2) - + # synchronize the mode of operation pool.set_size(0) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 779155a2a..f28aef4d4 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -12,15 +12,15 @@ from gitdb.stream import DeltaApplyReader from gitdb.pack import ( - PackEntity, - PackIndexFile, - PackFile - ) + PackEntity, + PackIndexFile, + PackFile +) from gitdb.base import ( - OInfo, - OStream, - ) + OInfo, + OStream, +) from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation @@ -39,15 +39,15 @@ def bin_sha_from_filename(filename): #} END utilities class TestPack(TestBase): - + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - - + + def _assert_index_file(self, index, version, size): assert index.packfile_checksum() != index.indexfile_checksum() assert len(index.packfile_checksum()) == 20 @@ -55,93 +55,93 @@ def _assert_index_file(self, index, version, size): assert index.version() == version assert index.size() == size assert len(index.offsets()) == size - + # get all data of all objects for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) - + entry = index.entry(oidx) assert len(entry) == 3 - + assert entry[0] == index.offset(oidx) assert entry[1] == sha assert entry[2] == index.crc(oidx) - + # verify partial sha for l in (4,8,11,17,20): assert index.partial_sha_to_index(sha[:l], l*2) == oidx - + # END for each object index in indexfile self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - - + + def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 assert pack.size() == size assert len(pack.checksum()) == 20 - + num_obj = 0 for obj in pack.stream_iter(): num_obj += 1 info = pack.info(obj.pack_offset) stream = pack.stream(obj.pack_offset) - + assert info.pack_offset == stream.pack_offset assert info.type_id == stream.type_id assert hasattr(stream, 'read') - + # it should be possible to read from both streams assert obj.read() == stream.read() - + streams = pack.collect_streams(obj.pack_offset) assert streams - + # read the stream try: dstream = DeltaApplyReader.new(streams) except ValueError: - # ignore these, old git versions use only ref deltas, + # ignore these, old git versions use only ref deltas, # which we havent resolved ( as we are without an index ) # Also ignore non-delta streams continue # END get deltastream - + # read all data = dstream.read() assert len(data) == dstream.size - + # test seek dstream.seek(0) assert dstream.read() == data - - + + # read chunks # NOTE: the current implementation is safe, it basically transfers # all calls to the underlying memory map - + # END for each object assert num_obj == size - - + + def test_pack_index(self): # check version 1 and 2 - for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): index = PackIndexFile(indexfile) self._assert_index_file(index, version, size) # END run tests - + def test_pack(self): - # there is this special version 3, but apparently its like 2 ... + # there is this special version 3, but apparently its like 2 ... for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): pack = PackFile(packfile) self._assert_pack_file(pack, version, size) # END for each pack to test - + @with_rw_directory def test_pack_entity(self, rw_dir): pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo @@ -150,7 +150,7 @@ def test_pack_entity(self, rw_dir): assert entity.pack().path() == packfile assert entity.index().path() == indexfile pack_objs.extend(entity.stream_iter()) - + count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): count += 1 @@ -158,10 +158,10 @@ def test_pack_entity(self, rw_dir): assert len(info.binsha) == 20 assert info.type_id == stream.type_id assert info.size == stream.size - + # we return fully resolved items, which is implied by the sha centric access assert not info.type_id in delta_types - + # try all calls assert len(entity.collect_streams(info.binsha)) oinfo = entity.info(info.binsha) @@ -170,7 +170,7 @@ def test_pack_entity(self, rw_dir): ostream = entity.stream(info.binsha) assert isinstance(ostream, OStream) assert ostream.binsha is not None - + # verify the stream try: assert entity.is_valid_stream(info.binsha, use_crc=True) @@ -180,16 +180,16 @@ def test_pack_entity(self, rw_dir): assert entity.is_valid_stream(info.binsha, use_crc=False) # END for each info, stream tuple assert count == size - + # END for each entity - + # pack writing - write all packs into one # index path can be None pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 def rewind_streams(): - for obj in pack_objs: + for obj in pack_objs: obj.stream.seek(0) #END utility for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): @@ -199,23 +199,23 @@ def rewind_streams(): ifile = open(ipath, 'wb') iwrite = ifile.write #END handle ip - + # make sure we rewind the streams ... we work on the same objects over and over again - if iteration > 0: + if iteration > 0: rewind_streams() #END rewind streams iteration += 1 - + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) pfile.close() assert os.path.getsize(ppath) > 100 - + # verify pack pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha - + # verify index if ipath is not None: ifile.close() @@ -227,7 +227,7 @@ def rewind_streams(): assert idx.size() == len(pack_objs) #END verify files exist #END for each packpath, indexpath pair - + # verify the packs throughly rewind_streams() entity = PackEntity.create(pack_objs, rw_dir) @@ -239,9 +239,9 @@ def rewind_streams(): # END for each crc mode #END for each info assert count == len(pack_objs) - - + + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets - # of course without really needing such a huge pack + # of course without really needing such a huge pack raise SkipTest() diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 6dc27463c..8360ea36c 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -32,18 +32,18 @@ class TestStream(TestBase): """Test stream classes""" - + data_sizes = (15, 10000, 1000*1024+512) - + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): - """Make stream tests - the orig_stream is seekable, allowing it to be + """Make stream tests - the orig_stream is seekable, allowing it to be rewound and reused :param cdata: the data we expect to read from stream, the contents :param rewind_stream: function called to rewind the stream to make it ready for reuse""" ns = 10 assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) - + # read in small steps ss = len(cdata) / ns for i in range(ns): @@ -55,30 +55,30 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): if rest: assert rest == cdata[-len(rest):] # END handle rest - + if isinstance(stream, DecompressMemMapReader): assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type - + rewind_stream(stream) - + # read everything rdata = stream.read() assert rdata == cdata - + if isinstance(stream, DecompressMemMapReader): assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type - + def test_decompress_reader(self): for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: cdata = make_bytes(ds, randomize=False) - + # zdata = zipped actual data # cdata = original content data - + # create reader if with_size: # need object data @@ -86,7 +86,7 @@ def test_decompress_reader(self): type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) assert type == str_blob_type - + # even if we don't set the size, it will be set automatically on first read test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) assert test_reader._s == len(cdata) @@ -95,60 +95,59 @@ def test_decompress_reader(self): zdata = zlib.compress(cdata) reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) assert reader._s == len(cdata) - # END get reader - + # END get reader + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) - + # put in a dummy stream for closing dummy = DummyStream() reader._m = dummy - + assert not dummy.closed del(reader) assert dummy.closed == close_on_deletion # END for each datasize # END whether size should be used # END whether stream should be closed when deleted - + def test_sha_writer(self): writer = Sha1Writer() assert 2 == writer.write("hi") assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 - + # make sure it does something ;) prev_sha = writer.sha() writer.write("hi again") assert writer.sha() != prev_sha - + def test_compressed_writer(self): for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) data = make_bytes(ds, randomize=False) - + # for now, just a single write, code doesn't care about chunking assert len(data) == ostream.write(data) ostream.close() - + # its closed already self.failUnlessRaises(OSError, os.close, fd) - + # read everything back, compare to data we zip fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) assert len(written_data) == os.path.getsize(path) os.close(fd) assert written_data == zlib.compress(data, 1) # best speed - + os.remove(path) # END for each os - + def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) - + # if there is a bug, we will be missing one byte exactly ! data = ostream.read() assert len(data) == ostream.size - diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 35f9f44a7..ed69f0d1f 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -8,28 +8,28 @@ from lib import TestBase from gitdb.util import ( - to_hex_sha, - to_bin_sha, - NULL_HEX_SHA, + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, LockedFD - ) +) + - class TestUtils(TestBase): def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA - + def _cmp_contents(self, file_path, data): - # raise if data from file at file_path + # raise if data from file at file_path # does not match data string fp = open(file_path, "rb") try: assert fp.read() == data finally: fp.close() - + def test_lockedfd(self): my_file = tempfile.mktemp() orig_data = "hello" @@ -37,43 +37,43 @@ def test_lockedfd(self): my_file_fp = open(my_file, "wb") my_file_fp.write(orig_data) my_file_fp.close() - + try: lfd = LockedFD(my_file) - lockfilepath = lfd._lockfilepath() - + lockfilepath = lfd._lockfilepath() + # cannot end before it was started self.failUnlessRaises(AssertionError, lfd.rollback) self.failUnlessRaises(AssertionError, lfd.commit) - + # open for writing assert not os.path.isfile(lockfilepath) wfd = lfd.open(write=True) assert lfd._fd is wfd assert os.path.isfile(lockfilepath) - + # write data and fail os.write(wfd, new_data) lfd.rollback() assert lfd._fd is None self._cmp_contents(my_file, orig_data) assert not os.path.isfile(lockfilepath) - + # additional call doesnt fail lfd.commit() lfd.rollback() - + # test reading lfd = LockedFD(my_file) rfd = lfd.open(write=False) assert os.read(rfd, len(orig_data)) == orig_data - + assert os.path.isfile(lockfilepath) # deletion rolls back del(lfd) assert not os.path.isfile(lockfilepath) - - + + # write data - concurrently lfd = LockedFD(my_file) olfd = LockedFD(my_file) @@ -82,17 +82,17 @@ def test_lockedfd(self): assert os.path.isfile(lockfilepath) # another one fails self.failUnlessRaises(IOError, olfd.open) - + wfdstream.write(new_data) lfd.commit() assert not os.path.isfile(lockfilepath) self._cmp_contents(my_file, new_data) - + # could test automatic _end_writing on destruction finally: os.remove(my_file) # END final cleanup - + # try non-existing file for reading lfd = LockedFD(tempfile.mktemp()) try: @@ -102,4 +102,3 @@ def test_lockedfd(self): else: self.fail("expected OSError") # END handle exceptions - diff --git a/gitdb/util.py b/gitdb/util.py index 1662b662d..b167b4d12 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -24,10 +24,9 @@ from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, SlidingWindowMapManager, + SlidingWindowMapBuffer +) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -60,7 +59,7 @@ def unpack_from(fmt, data, offset=0): #{ Globals -# A pool distributing tasks, initially with zero threads, hence everything +# A pool distributing tasks, initially with zero threads, hence everything # will be handled in the main thread pool = ThreadPool(0) @@ -97,35 +96,35 @@ def unpack_from(fmt, data, offset=0): #} END Aliases -#{ compatibility stuff ... +#{ compatibility stuff ... class _RandomAccessStringIO(object): - """Wrapper to provide required functionality in case memory maps cannot or may + """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' - + def __init__(self, buf=''): self._sio = StringIO(buf) - + def __getattr__(self, attr): return getattr(self._sio, attr) - + def __len__(self): return len(self.getvalue()) - + def __getitem__(self, i): return self.getvalue()[i] - + def __getslice__(self, start, end): return self.getvalue()[start:end] - + #} END compatibility stuff ... #{ Routines def make_sha(source=''): """A python2.4 workaround for the sha/hashlib module fiasco - + **Note** From the dulwich project """ try: return hashlib.sha1(source) @@ -138,25 +137,25 @@ def allocate_memory(size): if size == 0: return _RandomAccessStringIO('') # END handle empty chunks gracefully - + try: return mmap.mmap(-1, size) # read-write by default except EnvironmentError: # setup real memory instead # this of course may fail if the amount of memory is not available in - # one chunk - would only be the case in python 2.4, being more likely on + # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. return _RandomAccessStringIO("\0"*size) # END handle memory allocation - + def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd - + :param fd: file descriptor opened for reading :param stream: if False, random access is provided, otherwise the stream interface is provided. - :param allow_mmap: if True, its allowed to map the contents into memory, which + :param allow_mmap: if True, its allowed to map the contents into memory, which allows large files to be handled and accessed efficiently. The file-descriptor will change its position if this is False""" try: @@ -171,24 +170,24 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): except OSError: pass # END exception handling - + # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: return _RandomAccessStringIO(contents) return contents - + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible - + :return: random access compatible memory of the given filepath :param stream: see ``file_contents_ro`` :param allow_mmap: see ``file_contents_ro`` :param flags: additional flags to pass to os.open :raise OSError: If the file could not be opened - - **Note** for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: @@ -196,19 +195,19 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): finally: close(fd) # END assure file is closed - + def sliding_ro_buffer(filepath, flags=0): """ :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) - + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: return sha return bin_to_hex(sha) - + def to_bin_sha(sha): if len(sha) == 20: return sha @@ -227,12 +226,12 @@ class LazyMixin(object): is actually accessed and retrieved the first time. All future accesses will return the cached value as stored in the Instance's dict or slot. """ - + __slots__ = tuple() - + def __getattr__(self, attr): """ - Whenever an attribute is requested that we do not know, we allow it + Whenever an attribute is requested that we do not know, we allow it to be created and set. Next time the same attribute is reqeusted, it is simply returned from our dict/slots. """ self._set_cache_(attr) @@ -241,65 +240,65 @@ def __getattr__(self, attr): def _set_cache_(self, attr): """ - This method should be overridden in the derived class. + This method should be overridden in the derived class. It should check whether the attribute named by attr can be created and cached. Do nothing if you do not know the attribute or call your subclass - - The derived class may create as many additional attributes as it deems - necessary in case a git command returns more information than represented + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented in the single attribute.""" pass - + class LockedFD(object): """ This class facilitates a safe read and write operation to a file on disk. - If we write to 'file', we obtain a lock file at 'file.lock' and write to - that instead. If we succeed, the lock file will be renamed to overwrite + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite the original file. - - When reading, we obtain a lock file, but to prevent other writers from + + When reading, we obtain a lock file, but to prevent other writers from succeeding while we are reading the file. - - This type handles error correctly in that it will assure a consistent state + + This type handles error correctly in that it will assure a consistent state on destruction. - + **note** with this setup, parallel reading is not possible""" __slots__ = ("_filepath", '_fd', '_write') - + def __init__(self, filepath): """Initialize an instance with the givne filepath""" self._filepath = filepath self._fd = None self._write = None # if True, we write a file - + def __del__(self): # will do nothing if the file descriptor is already closed if self._fd is not None: self.rollback() - + def _lockfilepath(self): return "%s.lock" % self._filepath - + def open(self, write=False, stream=False): """ Open the file descriptor for reading or writing, both in binary mode. - + :param write: if True, the file descriptor will be opened for writing. Other wise it will be opened read-only. - :param stream: if True, the file descriptor will be wrapped into a simple stream + :param stream: if True, the file descriptor will be wrapped into a simple stream object which supports only reading or writing :return: fd to read from or write to. It is still maintained by this instance and must not be closed directly :raise IOError: if the lock could not be retrieved :raise OSError: If the actual file could not be opened for reading - + **note** must only be called once""" if self._write is not None: raise AssertionError("Called %s multiple times" % self.open) - + self._write = write - + # try to open the lock file binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary @@ -313,7 +312,7 @@ def open(self, write=False, stream=False): except OSError: raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) # END handle lock retrieval - + # open actual file if required if self._fd is None: # we could specify exlusive here, as we obtained the lock anyway @@ -325,7 +324,7 @@ def open(self, write=False, stream=False): raise # END handle lockfile # END open descriptor for reading - + if stream: # need delayed import from stream import FDStream @@ -333,33 +332,33 @@ def open(self, write=False, stream=False): else: return self._fd # END handle stream - + def commit(self): - """When done writing, call this function to commit your changes into the - actual file. + """When done writing, call this function to commit your changes into the + actual file. The file descriptor will be closed, and the lockfile handled. - + **Note** can be called multiple times""" self._end_writing(successful=True) - + def rollback(self): - """Abort your operation without any changes. The file descriptor will be + """Abort your operation without any changes. The file descriptor will be closed, and the lock released. - + **Note** can be called multiple times""" self._end_writing(successful=False) - + def _end_writing(self, successful=True): """Handle the lock according to the write mode """ if self._write is None: raise AssertionError("Cannot end operation if it wasn't started yet") - + if self._fd is None: return - + os.close(self._fd) self._fd = None - + lockfile = self._lockfilepath() if self._write and successful: # on windows, rename does not silently overwrite the existing one @@ -369,7 +368,7 @@ def _end_writing(self, successful=True): # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) - + # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well From b6c493deb2341fb843d71b66b2aa23078638755c Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:41:15 -0400 Subject: [PATCH 0274/3719] Pick off the low hanging fruit This fixes most of the import errors that came from using the implicit relative imports that Python 2 supports. This also fixes the use of `xrange`, which has replaced `range` in Python 3. The same has happened for `izip`, which is also being aliased. The octal number syntax changed in Python 3, so we are now converting from strings using the `int` built-in function, which will produce the same output across both versions of Python. --- gitdb/__init__.py | 7 ++- gitdb/base.py | 18 +++---- gitdb/db/__init__.py | 13 +++-- gitdb/db/base.py | 1 + gitdb/db/git.py | 18 +++---- gitdb/db/loose.py | 20 ++++---- gitdb/db/mem.py | 23 ++++++--- gitdb/db/pack.py | 14 +++--- gitdb/db/ref.py | 10 ++-- gitdb/exc.py | 2 +- gitdb/fun.py | 21 +++++++-- gitdb/pack.py | 94 ++++++++++++++++++++----------------- gitdb/stream.py | 43 ++++++++++------- gitdb/test/__init__.py | 2 +- gitdb/test/db/lib.py | 12 ++++- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_mem.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 11 ++++- gitdb/test/test_base.py | 2 +- gitdb/test/test_example.py | 12 +++-- gitdb/test/test_pack.py | 23 +++++---- gitdb/test/test_stream.py | 16 +++---- gitdb/test/test_util.py | 2 +- gitdb/util.py | 18 +++---- 27 files changed, 228 insertions(+), 164 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 847269a33..66d1b1c40 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -32,7 +32,6 @@ def _init_externals(): # default imports -from db import * -from base import * -from stream import * - +from gitdb.base import * +from gitdb.db import * +from gitdb.stream import * diff --git a/gitdb/base.py b/gitdb/base.py index a673c2376..1eb423284 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -3,15 +3,15 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" -from util import ( - bin_to_hex, - zlib - ) - -from fun import ( - type_id_to_type_map, - type_to_type_id_map - ) +from gitdb.util import ( + bin_to_hex, + zlib +) + +from gitdb.fun import ( + type_id_to_type_map, + type_to_type_id_map +) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index e5935b7c2..0a2a46a64 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -3,10 +3,9 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import * -from loose import * -from mem import * -from pack import * -from git import * -from ref import * - +from gitdb.db.base import * +from gitdb.db.loose import * +from gitdb.db.mem import * +from gitdb.db.pack import * +from gitdb.db.git import * +from gitdb.db.ref import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 0eef1e5d5..85df324f7 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -20,6 +20,7 @@ ) from itertools import chain +from functools import reduce __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 6e6ec5d1f..5c74a2049 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -2,15 +2,15 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - CompoundDB, - ObjectDBW, - FileDBBase - ) - -from loose import LooseObjectDB -from pack import PackedDB -from ref import ReferenceDB +from gitdb.db.base import ( + CompoundDB, + ObjectDBW, + FileDBBase +) + +from gitdb.db.loose import LooseObjectDB +from gitdb.db.pack import PackedDB +from gitdb.db.ref import ReferenceDB from gitdb.util import LazyMixin from gitdb.exc import ( diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4ebca84d3..ac1b9d1d6 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -2,11 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - FileDBBase, - ObjectDBR, - ObjectDBW - ) +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + ObjectDBW +) from gitdb.exc import ( @@ -69,11 +69,11 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): # On windows we need to keep it writable, otherwise it cannot be removed # either - new_objects_mode = 0444 + new_objects_mode = int("444", 8) if os.name == 'nt': - new_objects_mode = 0644 - - + new_objects_mode = int("644", 8) + + def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) self._hexsha_to_file = dict() @@ -133,7 +133,7 @@ def _map_loose_object(self, sha): db_path = self.db_path(self.object_path(bin_to_hex(sha))) try: return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) - except OSError,e: + except OSError as e: if e.errno != ENOENT: # try again without noatime try: diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index e4fba94b3..3847c34df 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -3,11 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains the MemoryDatabase implementation""" -from loose import LooseObjectDB -from base import ( - ObjectDBR, - ObjectDBW - ) +from gitdb.db.loose import LooseObjectDB +from gitdb.db.base import ( + ObjectDBR, + ObjectDBW +) from gitdb.base import ( OStream, @@ -19,7 +19,18 @@ UnsupportedOperation ) -from cStringIO import StringIO +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, +) + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO __all__ = ("MemoryDB", ) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 09f811847..eca02bbff 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -3,11 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" -from base import ( - FileDBBase, - ObjectDBR, - CachingDB - ) +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + CachingDB +) from gitdb.util import LazyMixin @@ -19,6 +19,8 @@ from gitdb.pack import PackEntity +from functools import reduce + import os import glob @@ -104,7 +106,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in xrange(index.size()): + for index in range(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 368ab9a61..748f7c145 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,9 +2,9 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - CompoundDB, - ) +from gitdb.db.base import ( + CompoundDB, +) import os __all__ = ('ReferenceDB', ) @@ -33,7 +33,7 @@ def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls if dbcls is None: # late import - from git import GitDB + from gitdb.db.git import GitDB dbcls = GitDB # END get db type @@ -68,7 +68,7 @@ def _update_dbs_from_ref_file(self): db.databases() # END verification self._dbs.append(db) - except Exception, e: + except Exception: # ignore invalid paths or issues pass # END for each path to add diff --git a/gitdb/exc.py b/gitdb/exc.py index 47fc80912..73f84d299 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with common exceptions""" -from util import to_hex_sha +from gitdb.util import to_hex_sha class ODBError(Exception): """All errors thrown by the object database""" diff --git a/gitdb/fun.py b/gitdb/fun.py index 9e5c44a6c..ce55438d1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -6,17 +6,28 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from exc import ( +from gitdb.exc import ( BadObjectType - ) +) -from util import zlib +from gitdb.util import zlib decompressobj = zlib.decompressobj import mmap -from itertools import islice, izip +from itertools import islice + +try: + from itertools import izip +except ImportError: + izip = zip -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO # INVARIANTS OFS_DELTA = 6 diff --git a/gitdb/pack.py b/gitdb/pack.py index 4a0badccb..aea0d1e5a 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -4,31 +4,32 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) -from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + BadObject, + UnsupportedOperation, + ParseError +) -from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) +from gitdb.util import ( + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, +) + +from gitdb.fun import ( + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size +) try: from _perf import PackIndexFile_sha_to_index @@ -36,22 +37,23 @@ pass # END try c module -from base import ( # Amazing ! - OInfo, - OStream, - OPackInfo, - OPackStream, - ODeltaStream, - ODeltaPackInfo, - ODeltaPackStream, - ) -from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) +from gitdb.base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, +) + +from gitdb.stream import ( + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer +) from struct import ( pack, @@ -60,7 +62,11 @@ from binascii import crc32 -from itertools import izip +try: + from itertools import izip +except ImportError: + izip = zip + import tempfile import array import os @@ -200,7 +206,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in xrange(255): + for i in range(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i+1] += v @@ -407,7 +413,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in xrange(self.size())) + return tuple(self.offset(index) for index in range(self.size())) # END handle version def sha_to_index(self, sha): @@ -694,7 +700,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in xrange(self._index.size()): + for index in range(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/stream.py b/gitdb/stream.py index b21c39c9d..52b54af50 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,28 +3,35 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + import errno import mmap import os -from fun import ( - msb_size, - stream_copy, - apply_delta_data, - connect_deltas, - DeltaChunkList, - delta_types - ) - -from util import ( - allocate_memory, - LazyMixin, - make_sha, - write, - close, - zlib - ) +from gitdb.fun import ( + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + DeltaChunkList, + delta_types +) + +from gitdb.util import ( + allocate_memory, + LazyMixin, + make_sha, + write, + close, + zlib +) has_perf_mod = False try: diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index e84e503b6..ca104c0c5 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -9,7 +9,7 @@ def _init_pool(): """Assure the pool is actually threaded""" size = 2 - print "Setting ThreadPool to %i" % size + print("Setting ThreadPool to %i" % size) gitdb.util.pool.set_size(size) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index dc89039dd..18b22ff21 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,15 @@ from gitdb.typ import str_blob_type from async import IteratorReader -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + from struct import pack @@ -40,7 +48,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in xrange(ni): + for i in range(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), StringIO(data)) new_istream = db.store(istream) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 4894c6a79..cce2b9c09 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index e295db563..5e42b639a 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index ac9bc34e9..9235b21d3 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import ( MemoryDB, LooseObjectDB diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 0d9110a01..f5a4dcb92 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import PackedDB from gitdb.test.lib import fixture_path diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 086446823..752c31de5 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import ReferenceDB from gitdb.util import ( diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 685af2fad..3ac7142c1 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -16,7 +16,14 @@ import sys import random from array import array -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO import glob import unittest @@ -109,7 +116,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes / 4 - producer = xrange(actual_size) + producer = range(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 76b4d2709..4cca7dabe 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" -from lib import ( +from gitdb.test.lib import ( TestBase, DummyStream, DeriveTest, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index f45063b1e..f57cc5029 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,12 +3,18 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from lib import * +from gitdb.test.lib import * from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool - -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO from async import IteratorReader diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index f28aef4d4..bcda3cfb8 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -3,12 +3,13 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" -from lib import ( - TestBase, - with_rw_directory, - with_packs_rw, - fixture_path - ) +from gitdb.test.lib import ( + TestBase, + with_rw_directory, + with_packs_rw, + fixture_path +) + from gitdb.stream import DeltaApplyReader from gitdb.pack import ( @@ -25,7 +26,13 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from itertools import izip, chain +from itertools import chain + +try: + from itertools import izip +except ImportError: + izip = zip + from nose import SkipTest import os @@ -57,7 +64,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in xrange(index.size()): + for oidx in range(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 8360ea36c..53aa8e21d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -3,14 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" -from lib import ( - TestBase, - DummyStream, - Sha1Writer, - make_bytes, - make_object, - fixture_path - ) +from gitdb.test.lib import ( + TestBase, + DummyStream, + Sha1Writer, + make_bytes, + make_object, + fixture_path +) from gitdb import * from gitdb.util import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index ed69f0d1f..4672dd604 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -6,7 +6,7 @@ import tempfile import os -from lib import TestBase +from gitdb.test.lib import TestBase from gitdb.util import ( to_hex_sha, to_bin_sha, diff --git a/gitdb/util.py b/gitdb/util.py index b167b4d12..3dcf3d7dc 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,13 +8,13 @@ import sys import errno -from cStringIO import StringIO - -# in py 2.4, StringIO is only StringI, without write support. -# Hence we must use the python implementation for this -if sys.version_info[1] < 5: - from StringIO import StringIO -# END handle python 2.4 +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO try: import async.mod.zlib as zlib @@ -303,7 +303,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode, 0600) + fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) if not write: os.close(fd) else: @@ -372,7 +372,7 @@ def _end_writing(self, successful=True): # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well - chmod(self._filepath, 0644) + chmod(self._filepath, int("644", 8)) else: # just delete the file so far, we failed os.remove(lockfile) From 5bfa7e6cb0872c815720f7e861393125ad0855e7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:45:51 -0400 Subject: [PATCH 0275/3719] Temporarily switch out async for testing This will be switched back when the pull request for Python 3 support has been merged into the central async repository. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 978105388..c28387570 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = https://github.com/gitpython-developers/async.git + url = https://github.com/kevin-brown/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git From 85f2b9baf1c6a5738967f8e77d2a65a1a842bde3 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:48:03 -0400 Subject: [PATCH 0276/3719] Test against Python 3.4 If it works in Python 3.3, it should also work in Python 3.4. Considering it is the latest stable release, gitdb should be tested against it. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ff263f20d..cf1d13666 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.6" - "2.7" - "3.3" + - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) script: nosetests From b881134ec816d2a54f6e8deced8db25b4bd5baa7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 16:10:54 -0400 Subject: [PATCH 0277/3719] Convert strings to bytes for PY3 In Python 3, the default string type is now the Python 2 unicode strings. The unicode strings cannot be converted to a byte stream, so we have to convert it before writing to the streams. --- gitdb/test/lib.py | 6 +++--- gitdb/test/test_stream.py | 9 ++++----- gitdb/test/test_util.py | 13 +++++++------ gitdb/util.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 3ac7142c1..f52cf79d4 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -54,7 +54,7 @@ def wrapper(self): try: return func(self, path) except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + sys.stderr.write("Test %s.%s failed, output is at %r\n" % (type(self).__name__, func.__name__, path)) keep = True raise finally: @@ -115,7 +115,7 @@ def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" - actual_size = size_in_bytes / 4 + actual_size = size_in_bytes // 4 producer = range(actual_size) if randomize: producer = list(producer) @@ -127,7 +127,7 @@ def make_bytes(size_in_bytes, randomize=False): def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) - return odata + data + return odata.encode("ascii") + data def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 53aa8e21d..f6eb371bd 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -3,6 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" + from gitdb.test.lib import ( TestBase, DummyStream, @@ -16,20 +17,18 @@ from gitdb.util import ( NULL_HEX_SHA, hex_to_bin - ) +) from gitdb.util import zlib from gitdb.typ import ( str_blob_type - ) +) import time import tempfile import os - - class TestStream(TestBase): """Test stream classes""" @@ -45,7 +44,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) # read in small steps - ss = len(cdata) / ns + ss = len(cdata) // ns for i in range(ns): data = stream.read(ss) chunk = cdata[i*ss:(i+1)*ss] diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 4672dd604..ec9a86ce7 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -5,6 +5,7 @@ """Test for object db""" import tempfile import os +import sys from gitdb.test.lib import TestBase from gitdb.util import ( @@ -19,14 +20,14 @@ class TestUtils(TestBase): def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA.encode("ascii") def _cmp_contents(self, file_path, data): # raise if data from file at file_path # does not match data string fp = open(file_path, "rb") try: - assert fp.read() == data + assert fp.read() == data.encode("ascii") finally: fp.close() @@ -35,7 +36,7 @@ def test_lockedfd(self): orig_data = "hello" new_data = "world" my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data) + my_file_fp.write(orig_data.encode("ascii")) my_file_fp.close() try: @@ -53,7 +54,7 @@ def test_lockedfd(self): assert os.path.isfile(lockfilepath) # write data and fail - os.write(wfd, new_data) + os.write(wfd, new_data.encode("ascii")) lfd.rollback() assert lfd._fd is None self._cmp_contents(my_file, orig_data) @@ -66,7 +67,7 @@ def test_lockedfd(self): # test reading lfd = LockedFD(my_file) rfd = lfd.open(write=False) - assert os.read(rfd, len(orig_data)) == orig_data + assert os.read(rfd, len(orig_data)) == orig_data.encode("ascii") assert os.path.isfile(lockfilepath) # deletion rolls back @@ -83,7 +84,7 @@ def test_lockedfd(self): # another one fails self.failUnlessRaises(IOError, olfd.open) - wfdstream.write(new_data) + wfdstream.write(new_data.encode("ascii")) lfd.commit() assert not os.path.isfile(lockfilepath) self._cmp_contents(my_file, new_data) diff --git a/gitdb/util.py b/gitdb/util.py index 3dcf3d7dc..5ea26d5bd 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -327,7 +327,7 @@ def open(self, write=False, stream=False): if stream: # need delayed import - from stream import FDStream + from gitdb.stream import FDStream return FDStream(self._fd) else: return self._fd From 01e40b5e02e90ccac06e3b0ec0adf1f8f4e48ebd Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 17:29:58 -0400 Subject: [PATCH 0278/3719] Use memoryview instead of buffer This uses memoryview by default, which is supported in Python 3 and Python 2.7, but not Python 2.6, and falls back to the old `buffer` type in Python 2.6 and when the memoryview does not support the type, such as when mmap instaces are passed in. --- gitdb/stream.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 52b54af50..a099eeba0 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -270,7 +270,10 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) + try: + indata = memoryview(self._m)[self._cws:self._cwe].tobytes() + except (NameError, TypeError): + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) From 0269405121d7ef065f7008c9c033e95e734f029a Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 18:20:48 -0400 Subject: [PATCH 0279/3719] Better handling of bytes This adds a `byte_ord` version of `ord` which will let `bytes` safely pass through in Python 3. `cmp` was also swapped out as it has been dropped in Python 3. --- gitdb/fun.py | 10 +++++----- gitdb/pack.py | 26 ++++++++++++++------------ gitdb/util.py | 15 ++++++++++++++- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index ce55438d1..8f2d46342 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -10,7 +10,7 @@ BadObjectType ) -from gitdb.util import zlib +from gitdb.util import byte_ord, zlib decompressobj = zlib.decompressobj import mmap @@ -411,13 +411,13 @@ def pack_object_header_info(data): The type_id should be interpreted according to the ``type_id_to_type_map`` map The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" - c = ord(data[0]) # first byte + c = byte_ord(data[0]) # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size while c & 0x80: - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 @@ -668,12 +668,12 @@ def is_equal_canonical_sha(canonical_length, match, sha1): hence the comparison will only use the last 4 bytes for uneven canonical representations :param match: less than 20 byte sha :param sha1: 20 byte sha""" - binary_length = canonical_length/2 + binary_length = canonical_length // 2 if match[:binary_length] != sha1[:binary_length]: return False if canonical_length - binary_length and \ - (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match)-1])) & 0xf0: return False # END handle uneven canonnical length return True diff --git a/gitdb/pack.py b/gitdb/pack.py index aea0d1e5a..4a9ee9ffa 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -15,6 +15,7 @@ LazyMixin, unpack_from, bin_to_hex, + byte_ord, ) from gitdb.fun import ( @@ -421,7 +422,7 @@ def sha_to_index(self, sha): :return: index usable with the ``offset`` or ``entry`` method, or None if the sha was not found in this pack index :param sha: 20 byte sha to lookup""" - first_byte = ord(sha[0]) + first_byte = byte_ord(sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: @@ -430,11 +431,11 @@ def sha_to_index(self, sha): # bisect until we have the sha while lo < hi: - mid = (lo + hi) / 2 - c = cmp(sha, get_sha(mid)) - if c < 0: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if sha < mid_sha: hi = mid - elif not c: + elif sha == mid_sha: return mid else: lo = mid + 1 @@ -453,7 +454,8 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - first_byte = ord(partial_bin_sha[0]) + first_byte = byte_ord(partial_bin_sha[0]) + get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: @@ -461,15 +463,15 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) + filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha)) # find lowest while lo < hi: - mid = (lo + hi) / 2 - c = cmp(filled_sha, get_sha(mid)) - if c < 0: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if filled_sha < mid_sha: hi = mid - elif not c: + elif filled_sha == mid_sha: # perfect match lo = mid break @@ -482,7 +484,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size(): + if lo + 1 < self.size(): next_sha = get_sha(lo+1) if next_sha and next_sha == cur_sha: raise AmbiguousObjectName(partial_bin_sha) diff --git a/gitdb/util.py b/gitdb/util.py index 5ea26d5bd..ed2dc3723 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -118,14 +118,27 @@ def __getitem__(self, i): def __getslice__(self, start, end): return self.getvalue()[start:end] +def byte_ord(b): + """ + Return the integer representation of the byte string. This supports Python + 3 byte arrays as well as standard strings. + """ + try: + return ord(b) + except TypeError: + return b + #} END compatibility stuff ... #{ Routines -def make_sha(source=''): +def make_sha(source=None): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ + if source is None: + source = "".encode("ascii") + try: return hashlib.sha1(source) except NameError: From d8405ee00bfc1ebdae7c41b45f8c374902a3d2b4 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 19:55:00 -0400 Subject: [PATCH 0280/3719] More bytes handling --- gitdb/db/base.py | 10 ++++++++++ gitdb/fun.py | 2 +- gitdb/pack.py | 6 +++++- gitdb/stream.py | 4 ++-- gitdb/test/db/test_ref.py | 2 +- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 85df324f7..aa7a7ee30 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,6 +22,8 @@ from itertools import chain from functools import reduce +import sys + __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -176,6 +178,14 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" + if sys.version_info[0] == 3: + text_type = str + else: + text_type = basestring + + if not isinstance(rela_path, text_type): + rela_path = rela_path.decode("utf-8") + return join(self._root_path, rela_path) #} END interface diff --git a/gitdb/fun.py b/gitdb/fun.py index 8f2d46342..9baeac3ec 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -402,7 +402,7 @@ def loose_object_header_info(m): :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0")].split(" ") + type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) return type_name, int(size) def pack_object_header_info(data): diff --git a/gitdb/pack.py b/gitdb/pack.py index 4a9ee9ffa..ad5eccb8b 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -121,7 +121,11 @@ def pack_object_at(cursor, offset, as_stream): abs_data_offset = offset + total_rela_offset if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + try: + buff = memoryview(data)[total_rela_offset:].tobytes() + except (NameError, TypeError): + buff = buffer(data, total_rela_offset) + stream = DecompressMemMapReader(buff, False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: diff --git a/gitdb/stream.py b/gitdb/stream.py index a099eeba0..75993db51 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -105,8 +105,8 @@ def _parse_header_info(self): maxb = 512 # should really be enough, cgit uses 8192 I believe self._s = maxb hdr = self.read(maxb) - hdrend = hdr.find("\0") - type, size = hdr[:hdrend].split(" ") + hdrend = hdr.find("\0".encode("ascii")) + type, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 752c31de5..e303fe2f8 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -19,7 +19,7 @@ def make_alt_file(self, alt_path, alt_list): The list can be empty""" alt_file = open(alt_path, "wb") for alt in alt_list: - alt_file.write(alt + "\n") + alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) alt_file.close() @with_rw_directory From f0b3e7bcc5278208294f40aa580ccb378ed1c165 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 20:14:24 -0400 Subject: [PATCH 0281/3719] Bytes for everyone! --- gitdb/db/loose.py | 8 ++++---- gitdb/stream.py | 2 +- gitdb/test/db/test_ref.py | 3 +-- gitdb/test/test_stream.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index ac1b9d1d6..656841651 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -16,10 +16,10 @@ ) from gitdb.stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - FDStream, - Sha1Writer + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer ) from gitdb.base import ( diff --git a/gitdb/stream.py b/gitdb/stream.py index 75993db51..1bd7fe2b2 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -551,7 +551,7 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" + :return: length of incoming data""" self.sha1.update(data) return len(data) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index e303fe2f8..a1387ee26 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -24,7 +24,7 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - NULL_BIN_SHA = '\0' * 20 + NULL_BIN_SHA = '\0'.encode("ascii") * 20 alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) @@ -35,7 +35,6 @@ def test_writing(self, path): # try empty, non-existing assert not rdb.has_object(NULL_BIN_SHA) - # setup alternate file # add two, one is invalid own_repo_path = fixture_path('../../../.git/objects') # use own repo diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f6eb371bd..f409f1788 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -111,13 +111,13 @@ def test_decompress_reader(self): def test_sha_writer(self): writer = Sha1Writer() - assert 2 == writer.write("hi") + assert 2 == writer.write("hi".encode("ascii")) assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 # make sure it does something ;) prev_sha = writer.sha() - writer.write("hi again") + writer.write("hi again".encode("ascii")) assert writer.sha() != prev_sha def test_compressed_writer(self): From a19a169ffd81e21f472dee8a9a38ac1c9fab9bd7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 21:04:47 -0400 Subject: [PATCH 0282/3719] Can't compare memoryview instances, convert to bytes --- gitdb/pack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index ad5eccb8b..d8f32c1c2 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -118,7 +118,6 @@ def pack_object_at(cursor, offset, as_stream): # assume its a base object total_rela_offset = data_rela_offset # END handle type id - abs_data_offset = offset + total_rela_offset if as_stream: try: @@ -129,6 +128,8 @@ def pack_object_at(cursor, offset, as_stream): if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: + if hasattr(delta_info, "tobytes"): + delta_info = delta_info.tobytes() return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: From 087803ee30456c4942d9c18d82c1d686eb081a27 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 21:49:51 -0400 Subject: [PATCH 0283/3719] Making a bit of progress... This changes the internals to use BytesIO over StringIO, which fixed a few of the failing tests in Python 3. We are only importing from `io` now, instead of the entire chain, as this is available in Python 2.6+. --- gitdb/db/mem.py | 8 +------- gitdb/fun.py | 10 ++-------- gitdb/stream.py | 26 ++++++++++---------------- gitdb/test/db/lib.py | 10 ++-------- gitdb/test/lib.py | 8 +------- gitdb/test/test_example.py | 8 +------- gitdb/test/test_stream.py | 4 ++-- gitdb/util.py | 8 +------- 8 files changed, 20 insertions(+), 62 deletions(-) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 3847c34df..1a2378f08 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -24,13 +24,7 @@ DecompressMemMapReader, ) -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO __all__ = ("MemoryDB", ) diff --git a/gitdb/fun.py b/gitdb/fun.py index 9baeac3ec..1d835ab35 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -21,13 +21,7 @@ except ImportError: izip = zip -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO # INVARIANTS OFS_DELTA = 6 @@ -453,7 +447,7 @@ def msb_size(data, offset=0): l = len(data) hit_msb = False while i < l: - c = ord(data[i+offset]) + c = byte_ord(data[i+offset]) size |= (c & 0x7f) << i*7 i += 1 if not c & 0x80: diff --git a/gitdb/stream.py b/gitdb/stream.py index 1bd7fe2b2..9b451506e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,13 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import BytesIO, StringIO import errno import mmap @@ -106,20 +100,20 @@ def _parse_header_info(self): self._s = maxb hdr = self.read(maxb) hdrend = hdr.find("\0".encode("ascii")) - type, size = hdr[:hdrend].split(" ".encode("ascii")) + typ, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size # adjust internal state to match actual header length that we ignore # The buffer will be depleted first on future reads self._br = 0 - hdrend += 1 # count terminating \0 - self._buf = StringIO(hdr[hdrend:]) + hdrend += 1 + self._buf = BytesIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend self._phi = True - return type, size + return typ.decode("ascii"), size #{ Interface @@ -133,8 +127,8 @@ def new(self, m, close_on_deletion=False): :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) - type, size = inst._parse_header_info() - return type, size, inst + typ, size = inst._parse_header_info() + return typ, size, inst def data(self): """:return: random access compatible data we are working on""" @@ -211,14 +205,14 @@ def read(self, size=-1): # END clamp size if size == 0: - return str() + return bytes() # END handle depletion # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream - dat = str() + dat = bytes() if self._buf: if self._buflen >= size: # have enough data @@ -588,7 +582,7 @@ class ZippedStoreShaWriter(Sha1Writer): __slots__ = ('buf', 'zip') def __init__(self): Sha1Writer.__init__(self) - self.buf = StringIO() + self.buf = BytesIO() self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) def __getattr__(self, attr): diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 18b22ff21..15fbf3fc3 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -24,13 +24,7 @@ from async import IteratorReader -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO from struct import pack @@ -41,7 +35,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world" + two_lines = "1234\nhello world".encode("ascii") all_data = (two_lines, ) def _assert_object_writing_simple(self, db): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index f52cf79d4..75342f173 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -17,13 +17,7 @@ import random from array import array -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO import glob import unittest diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index f57cc5029..c714f107f 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -8,13 +8,7 @@ from gitdb.db import LooseObjectDB from gitdb.util import pool -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO from async import IteratorReader diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f409f1788..92755d9aa 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -82,9 +82,9 @@ def test_decompress_reader(self): if with_size: # need object data zdata = zlib.compress(make_object(str_blob_type, cdata)) - type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + typ, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) - assert type == str_blob_type + assert typ == str_blob_type # even if we don't set the size, it will be set automatically on first read test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) diff --git a/gitdb/util.py b/gitdb/util.py index ed2dc3723..30c7008f4 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,13 +8,7 @@ import sys import errno -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO try: import async.mod.zlib as zlib From 0cf09d3310cba7f33b9ebc9badf61ab721d12857 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 22:15:09 -0400 Subject: [PATCH 0284/3719] Fix Python 2 failures --- gitdb/db/loose.py | 4 ++-- gitdb/db/mem.py | 4 ++-- gitdb/fun.py | 2 +- gitdb/test/db/lib.py | 11 ++++++----- gitdb/test/test_example.py | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 656841651..1b8fe643c 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -161,8 +161,8 @@ def set_ostream(self, stream): def info(self, sha): m = self._map_loose_object(sha) try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) + typ, size = loose_object_header_info(m) + return OInfo(sha, typ, size) finally: m.close() # END assure release of system resources diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1a2378f08..efd85af27 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -24,7 +24,7 @@ DecompressMemMapReader, ) -from io import StringIO +from io import BytesIO __all__ = ("MemoryDB", ) @@ -104,7 +104,7 @@ def stream_copy(self, sha_iter, odb): ostream = self.stream(sha) # compressed data including header - sio = StringIO(ostream.stream.data()) + sio = BytesIO(ostream.stream.data()) istream = IStream(ostream.type, ostream.size, sio, sha) odb.store(istream) diff --git a/gitdb/fun.py b/gitdb/fun.py index 1d835ab35..69e9826d1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -397,7 +397,7 @@ def loose_object_header_info(m): decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) - return type_name, int(size) + return type_name.decode("ascii"), int(size) def pack_object_header_info(data): """ diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 15fbf3fc3..a6cdbbe65 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -24,7 +24,7 @@ from async import IteratorReader -from io import StringIO +from io import BytesIO from struct import pack @@ -44,7 +44,7 @@ def _assert_object_writing_simple(self, db): ni = 250 for i in range(ni): data = pack(">L", i) - istream = IStream(str_blob_type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) assert new_istream is istream assert db.has_object(istream.binsha) @@ -82,7 +82,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(str_blob_type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -132,8 +132,9 @@ def _assert_object_writing_async(self, db): ni = 5000 def istream_generator(offset=0, ni=ni): for data_src in xrange(ni): - data = str(data_src + offset) - yield IStream(str_blob_type, len(data), StringIO(data)) + print(type(data_src), type(offset)) + data = bytes(data_src + offset) + yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item # END generator utility diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index c714f107f..c644b8849 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -8,7 +8,7 @@ from gitdb.db import LooseObjectDB from gitdb.util import pool -from io import StringIO +from io import BytesIO from async import IteratorReader @@ -33,8 +33,8 @@ def test_base(self): pass # END ignore exception if there are no loose objects - data = "my data" - istream = IStream("blob", len(data), StringIO(data)) + data = "my data".encode("ascii") + istream = IStream("blob", len(data), BytesIO(data)) # the object does not yet have a sha assert istream.binsha is None From c544101eed30d5656746080204f53a2563b3d535 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 12:45:15 +0200 Subject: [PATCH 0285/3719] Added sublime-text project As relative paths are used througout, it will work for everyone using sublime text out of the box. --- .gitignore | 1 + etc/sublime-text/git-python.sublime-project | 71 +++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 etc/sublime-text/git-python.sublime-project diff --git a/.gitignore b/.gitignore index df821cfa8..1a26c03a1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ cover/ /dist /doc/_build nbproject +*.sublime-workspace diff --git a/etc/sublime-text/git-python.sublime-project b/etc/sublime-text/git-python.sublime-project new file mode 100644 index 000000000..5d981925a --- /dev/null +++ b/etc/sublime-text/git-python.sublime-project @@ -0,0 +1,71 @@ +{ + "folders": + [ + // GIT-PYTHON + ///////////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + "git/ext" + ] + }, + // GITDB + //////// + { + "follow_symlinks": true, + "path": "../../git/ext/gitdb", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + "gitdb/ext" + ] + }, + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../../git/ext/gitdb/gitdb/ext/smmap", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + // ASYNC + //////// + { + "follow_symlinks": true, + "path": "../../git/ext/gitdb/gitdb/ext/async", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 9dc111e1aa8358aa39a35d5a169335bacce53646 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 13:01:54 +0200 Subject: [PATCH 0286/3719] Added sublime-text project Suitable for everyone thanks to relative paths --- .gitignore | 2 + etc/sublime-text/gitdb.sublime-project | 54 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 etc/sublime-text/gitdb.sublime-project diff --git a/.gitignore b/.gitignore index 1e097f6f8..c6247dbb0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dist/ *.pyc *.o *.so +.noseids +*.sublime-workspace \ No newline at end of file diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project new file mode 100644 index 000000000..bc0e37f0a --- /dev/null +++ b/etc/sublime-text/gitdb.sublime-project @@ -0,0 +1,54 @@ +{ + "folders": + [ + // GITDB + //////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + "gitdb/ext" + ] + }, + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../../gitdb/ext/smmap", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + // ASYNC + //////// + { + "follow_symlinks": true, + "path": "../../gitdb/ext/async", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 1af4b42a2354acbb53c7956d647655922658fd80 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 13:05:41 +0200 Subject: [PATCH 0287/3719] Added sublime-text project Relative paths will make it work for everyone right away --- .gitignore | 2 ++ etc/sublime-text/smmap.sublime-project | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 etc/sublime-text/smmap.sublime-project diff --git a/.gitignore b/.gitignore index 73b0b6188..01247547d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ dist/ MANIFEST .tox *.egg-info +.noseids +*.sublime-workspace diff --git a/etc/sublime-text/smmap.sublime-project b/etc/sublime-text/smmap.sublime-project new file mode 100644 index 000000000..251ebbd28 --- /dev/null +++ b/etc/sublime-text/smmap.sublime-project @@ -0,0 +1,21 @@ +{ + "folders": + [ + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 0465cf327d232101b2de69d714a468b7e1a66a74 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 16 Jul 2014 20:15:31 -0400 Subject: [PATCH 0288/3719] Start up compat and encoding files There were a few things which were being reused consistently for compatibility purposes, such as the `buffer`/`memoryview` functions as well as the `izip` method which needed to be aliased for Python 3. The `buffer` function was taken from `smmap` [1] and reworked slightly to handle the optional third parameter. This also adds a compatibility file dedicated entirely to encoding issues, which seem to be the biggest problem. The main functions were taken in part from the Django project [2] and rewritten slightly because our needs are a bit more narrow. A constants file has been added to consistently handle the constants which are required for the gitdb project in the core and the tests. This is part of a greater plan to reorganize the `util.py` file included in this project. This points the async extension back at the original repository and points it to the latest commit. [1]: https://github.com/Byron/smmap/blob/1af4b42a2354acbb53c7956d647655922658fd80/smmap/util.py#L20-L26 [2]: https://github.com/django/django/blob/b8d255071ead897cf68120cd2fae7c91326ca2cc/django/utils/encoding.py --- .gitmodules | 2 +- gitdb/const.py | 5 +++++ gitdb/db/base.py | 10 ++-------- gitdb/db/loose.py | 4 +++- gitdb/ext/async | 2 +- gitdb/fun.py | 13 +++++++------ gitdb/pack.py | 18 +++++++----------- gitdb/stream.py | 24 ++++++++++++++++++------ gitdb/test/db/lib.py | 8 +++----- gitdb/util.py | 10 +++------- gitdb/utils/__init__.py | 0 gitdb/utils/compat.py | 39 +++++++++++++++++++++++++++++++++++++++ gitdb/utils/encoding.py | 35 +++++++++++++++++++++++++++++++++++ 13 files changed, 124 insertions(+), 46 deletions(-) create mode 100644 gitdb/const.py create mode 100644 gitdb/utils/__init__.py create mode 100644 gitdb/utils/compat.py create mode 100644 gitdb/utils/encoding.py diff --git a/.gitmodules b/.gitmodules index c28387570..978105388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = https://github.com/kevin-brown/async.git + url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git diff --git a/gitdb/const.py b/gitdb/const.py new file mode 100644 index 000000000..147f79cb9 --- /dev/null +++ b/gitdb/const.py @@ -0,0 +1,5 @@ +from gitdb.utils.encoding import force_bytes + +NULL_BYTE = force_bytes("\0") +NULL_HEX_SHA = "0" * 40 +NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index aa7a7ee30..53a94d249 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -178,15 +178,9 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" - if sys.version_info[0] == 3: - text_type = str - else: - text_type = basestring - - if not isinstance(rela_path, text_type): - rela_path = rela_path.decode("utf-8") + from gitdb.utils.encoding import force_text - return join(self._root_path, rela_path) + return join(self._root_path, force_text(rela_path)) #} END interface diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 1b8fe643c..f66d6275f 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -51,6 +51,8 @@ stream_copy ) +from gitdb.utils.compat import MAXSIZE + import tempfile import mmap import sys @@ -200,7 +202,7 @@ def store(self, istream): if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version - stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) + stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, diff --git a/gitdb/ext/async b/gitdb/ext/async index 339024bfb..3f26b05c2 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 339024bfb1d0a2b091e63d7a7ea23a1c63189f5c +Subproject commit 3f26b05c2f1a079d5807ed15c01b053ee846e745 diff --git a/gitdb/fun.py b/gitdb/fun.py index 69e9826d1..2749f37d0 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,10 +16,7 @@ import mmap from itertools import islice -try: - from itertools import izip -except ImportError: - izip = zip +from gitdb.utils.compat import izip from io import StringIO @@ -394,10 +391,14 @@ def loose_object_header_info(m): :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" + from gitdb.const import NULL_BYTE + from gitdb.utils.encoding import force_text + decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) - return type_name.decode("ascii"), int(size) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) + + return force_text(type_name), int(size) def pack_object_header_info(data): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index d8f32c1c2..4e83ba39d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -63,10 +63,8 @@ from binascii import crc32 -try: - from itertools import izip -except ImportError: - izip = zip +from gitdb.const import NULL_BYTE +from gitdb.utils.compat import izip, buffer import tempfile import array @@ -90,6 +88,8 @@ def pack_object_at(cursor, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" + from gitdb.utils.encoding import force_bytes + data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -120,16 +120,12 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - try: - buff = memoryview(data)[total_rela_offset:].tobytes() - except (NameError, TypeError): - buff = buffer(data, total_rela_offset) + buff = buffer(data, total_rela_offset) stream = DecompressMemMapReader(buff, False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - if hasattr(delta_info, "tobytes"): - delta_info = delta_info.tobytes() + delta_info = force_bytes(delta_info) return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: @@ -468,7 +464,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha)) + filled_sha = partial_bin_sha + NULL_BYTE * (20 - len(partial_bin_sha)) # find lowest while lo < hi: diff --git a/gitdb/stream.py b/gitdb/stream.py index 9b451506e..9e49f50e0 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -27,6 +27,10 @@ zlib ) +from gitdb.const import NULL_BYTE +from gitdb.utils.compat import buffer +from gitdb.utils.encoding import force_bytes, force_text + has_perf_mod = False try: from _perf import apply_delta as c_apply_delta @@ -99,7 +103,7 @@ def _parse_header_info(self): maxb = 512 # should really be enough, cgit uses 8192 I believe self._s = maxb hdr = self.read(maxb) - hdrend = hdr.find("\0".encode("ascii")) + hdrend = hdr.find(NULL_BYTE) typ, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size @@ -113,7 +117,7 @@ def _parse_header_info(self): self._phi = True - return typ.decode("ascii"), size + return force_text(typ), size #{ Interface @@ -264,10 +268,7 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - try: - indata = memoryview(self._m)[self._cws:self._cwe].tobytes() - except (NameError, TypeError): - indata = buffer(self._m, self._cws, self._cwe - self._cws) + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) @@ -379,6 +380,7 @@ def _set_cache_too_slow_without_c(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" + from gitdb.utils.compat import buffer # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the @@ -546,7 +548,10 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written :return: length of incoming data""" + + data = force_bytes(data) self.sha1.update(data) + return len(data) # END stream interface @@ -589,8 +594,11 @@ def __getattr__(self, attr): return getattr(self.buf, attr) def write(self, data): + data = force_bytes(data) + alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) + return alen def close(self): @@ -630,11 +638,15 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written :return: lenght of incoming data""" + data = force_bytes(data) + self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): raise self.exc + return len(data) def close(self): diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index a6cdbbe65..d0028b897 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -35,7 +35,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world".encode("ascii") + two_lines = "1234\nhello world" all_data = (two_lines, ) def _assert_object_writing_simple(self, db): @@ -81,8 +81,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - - istream = IStream(str_blob_type, len(data), BytesIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii"))) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -131,8 +130,7 @@ def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - print(type(data_src), type(offset)) + for data_src in range(ni): data = bytes(data_src + offset) yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item diff --git a/gitdb/util.py b/gitdb/util.py index 30c7008f4..5a82c553e 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -30,10 +30,7 @@ mman = SlidingWindowMapManager() #END handle mman -try: - import hashlib -except ImportError: - import sha +import hashlib try: from struct import unpack_from @@ -84,9 +81,8 @@ def unpack_from(fmt, data, offset=0): close = os.close fsync = os.fsync -# constants -NULL_HEX_SHA = "0"*40 -NULL_BIN_SHA = "\0"*20 +# Backwards compatibility imports +from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA #} END Aliases diff --git a/gitdb/utils/__init__.py b/gitdb/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py new file mode 100644 index 000000000..b9da683fa --- /dev/null +++ b/gitdb/utils/compat.py @@ -0,0 +1,39 @@ +import sys + +PY3 = sys.version_info[0] == 3 + +try: + from itertools import izip +except ImportError: + izip = zip + +try: + # Python 2 + buffer = buffer + memoryview = buffer +except NameError: + # Python 3 has no `buffer`; only `memoryview` + def buffer(obj, offset, size=None): + if size is None: + return memoryview(obj)[offset:] + else: + return memoryview(obj[offset:offset+size]) + + memoryview = memoryview + +if PY3: + MAXSIZE = sys.maxsize +else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py new file mode 100644 index 000000000..12164e756 --- /dev/null +++ b/gitdb/utils/encoding.py @@ -0,0 +1,35 @@ +from gitdb.utils import compat + +if compat.PY3: + string_types = (str, ) + text_type = str +else: + string_types = (basestring, ) + text_type = unicode + +def force_bytes(data, encoding="utf-8"): + if isinstance(data, bytes): + return data + + if isinstance(data, compat.memoryview): + return bytes(data) + + if isinstance(data, string_types): + return data.encode(encoding) + + return data + +def force_text(data, encoding="utf-8"): + if isinstance(data, text_type): + return data + + if isinstance(data, string_types): + return data.decode(encoding) + + if not isinstance(data, bytes): + data = force_bytes(data, encoding) + + if compat.PY3: + return text_type(data, encoding) + else: + return text_type(data) From ecdae96cdb8bbde322f59fd119dab302e7797c18 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 16 Jul 2014 20:36:02 -0400 Subject: [PATCH 0289/3719] Fixed a few more encoding issues Bytes should always be returned from the streams, so the tests should be checking against byte strings instead of text strings. This also fixes the `sha_iter` as it relied on the Python 2 `iterkeys` which has been renamed to `keys` in Python 3. --- gitdb/db/loose.py | 3 ++- gitdb/db/mem.py | 5 ++++- gitdb/test/db/lib.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index f66d6275f..63f96352b 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -52,6 +52,7 @@ ) from gitdb.utils.compat import MAXSIZE +from gitdb.utils.encoding import force_bytes import tempfile import mmap @@ -116,7 +117,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha): :raise BadObject: """ candidate = None for binsha in self.sha_iter(): - if bin_to_hex(binsha).startswith(partial_hexsha): + if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)): # it can't ever find the same object twice if candidate is not None: raise AmbiguousObjectName(partial_hexsha) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index efd85af27..a22454685 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -86,7 +86,10 @@ def size(self): return len(self._cache) def sha_iter(self): - return self._cache.iterkeys() + try: + return self._cache.iterkeys() + except AttributeError: + return self._cache.keys() #{ Interface diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index d0028b897..38747deb2 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -21,6 +21,7 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type +from gitdb.utils.encoding import force_bytes from async import IteratorReader @@ -97,7 +98,7 @@ def _assert_object_writing(self, db): assert info.size == len(data) ostream = db.stream(sha) - assert ostream.read() == data + assert ostream.read() == force_bytes(data) assert ostream.type == str_blob_type assert ostream.size == len(data) else: From 5244f7751c10865df94980817cfbe99b7933d4d6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jul 2014 08:34:48 +0200 Subject: [PATCH 0290/3719] Untested fix for #172 See https://github.com/gitpython-developers/GitPython/issues/172 for more information --- git/diff.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git/diff.py b/git/diff.py index 8a4819ab1..e90fc1cfe 100644 --- a/git/diff.py +++ b/git/diff.py @@ -75,6 +75,10 @@ def diff(self, other=Index, paths=None, create_patch=False, **kwargs): args.append("-M") # check for renames else: args.append("--raw") + + # in any way, assure we don't see colored output, + # fixes https://github.com/gitpython-developers/GitPython/issues/172 + args.append('--no-color') if paths is not None and not isinstance(paths, (tuple,list)): paths = [ paths ] From c63dcacbfa996b1d0d81d50c359fa37e4906cfb1 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Mon, 21 Jul 2014 20:38:17 -0400 Subject: [PATCH 0291/3719] Convert types to bytes This makes it easier to deal with things internally as now everything is passed as bytes. --- gitdb/fun.py | 32 +++++++++++++++++++------------- gitdb/stream.py | 4 ++-- gitdb/typ.py | 12 +++++------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 2749f37d0..6495359bc 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -17,6 +17,12 @@ from itertools import islice from gitdb.utils.compat import izip +from gitdb.typ import ( + str_blob_type, + str_commit_type, + str_tree_type, + str_tag_type, +) from io import StringIO @@ -27,23 +33,23 @@ type_id_to_type_map = { 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", + 1 : str_commit_type, + 2 : str_tree_type, + 3 : str_blob_type, + 4 : str_tag_type, 5 : "", # EXT 2 OFS_DELTA : "OFS_DELTA", # OFFSET DELTA REF_DELTA : "REF_DELTA" # REFERENCE DELTA } -type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA -) +type_to_type_id_map = { + str_commit_type: 1, + str_tree_type: 2, + str_blob_type: 3, + str_tag_type: 4, + "OFS_DELTA": OFS_DELTA, + "REF_DELTA": REF_DELTA, +} # used when dealing with larger streams chunk_size = 1000 * mmap.PAGESIZE @@ -398,7 +404,7 @@ def loose_object_header_info(m): hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) - return force_text(type_name), int(size) + return type_name, int(size) def pack_object_header_info(data): """ diff --git a/gitdb/stream.py b/gitdb/stream.py index 9e49f50e0..5dd38ec97 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -29,7 +29,7 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import buffer -from gitdb.utils.encoding import force_bytes, force_text +from gitdb.utils.encoding import force_bytes has_perf_mod = False try: @@ -117,7 +117,7 @@ def _parse_header_info(self): self._phi = True - return force_text(typ), size + return typ, size #{ Interface diff --git a/gitdb/typ.py b/gitdb/typ.py index e84dd2455..edd1f2731 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,11 +4,9 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -#{ String types +from gitdb.utils.encoding import force_bytes -str_blob_type = "blob" -str_commit_type = "commit" -str_tree_type = "tree" -str_tag_type = "tag" - -#} END string types +str_blob_type = force_bytes("blob") +str_commit_type = force_bytes("commit") +str_tree_type = force_bytes("tree") +str_tag_type = force_bytes("tag") From 9421cbc67e81629895ff1f7f397254bfe096ba49 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Thu, 24 Jul 2014 12:39:27 +0200 Subject: [PATCH 0292/3719] Update README.md to use fixed date Relative dates are not that precise, so instead of 3 years ago set it the last release date to July 2011. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 818e37515..d2a858bf9 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ A distribution package can be obtained for manual installation at: [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) [![Coverage Status](https://coveralls.io/repos/gitpython-developers/GitPython/badge.png)](https://coveralls.io/r/gitpython-developers/GitPython) -The project was idle for 2 years, the last release was made about 3 years ago. Reason for this might have been the project's dependency on me as sole active maintainer, which is an issue in itself. +The project was idle for 2 years, the last release (v0.3.2 RC1) was made on July 2011. Reason for this might have been the project's dependency on me as sole active maintainer, which is an issue in itself. Now I am back and fully dedicated to pushing [OSS](https://github.com/Byron/bcore) forward in the realm of [digital content creation](http://gooseberry.blender.org/), and git-python will see some of my time as well. Therefore it will be moving forward, slowly but steadily. From dbd784b870a878ef6dbecd14310018cdaeda5c6d Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Fri, 25 Jul 2014 10:54:44 +0200 Subject: [PATCH 0293/3719] List runtime dependencies in requirements.txt More and more packages are listing their dependencies in requirements.txt which make it trivial to maintain and install them. --- MANIFEST.in | 1 + README.md | 10 +++++++--- requirements.txt | 2 ++ setup.py | 4 +++- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 89f5b92d0..95b2e883f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE include CHANGES include AUTHORS include README +include requirements.txt graft git/test/fixtures graft git/test/performance diff --git a/README.md b/README.md index d2a858bf9..0b5f5a2a0 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,20 @@ The object database implementation is optimized for handling large quantities of * Mock by Michael Foord used for tests - Tested with 1.0.1 +The list of dependencies are listed in /requirements.txt. The installer takes care of installing them for you though. + ### INSTALL If you have downloaded the source code: python setup.py install - -or if you want to obtain a copy more easily: + +or if you want to obtain a copy from the Pypi repository: pip install gitpython - + +Both commands will install the required package dependencies. + A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..67f5eb742 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# Remember to update README.md +gitdb>=0.5.1 diff --git a/setup.py b/setup.py index e7c927b13..58a763e49 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ VERSION = v.readline().strip() v.close() +with open('requirements.txt') as reqs_file: + requirements = reqs_file.read().splitlines() class build_py(_build_py): def run(self): @@ -73,7 +75,7 @@ def _stamp_version(filename): package_data = {'git.test' : ['fixtures/*']}, package_dir = {'git':'git'}, license = "BSD License", - install_requires='gitdb >= 0.5.1', + install_requires=requirements, zip_safe=False, long_description = """\ GitPython is a python library used to interact with Git repositories""", From 2ddd5e5ef89da7f1e3b3a7d081fbc7f5c46ac11c Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Fri, 25 Jul 2014 11:01:30 +0200 Subject: [PATCH 0294/3719] Use tox to easily run tests in venv tox https://pypi.python.org/pypi/tox is a thin wrapper around virtualenv which let you craft a fresh python environement to execute command in. It creates the env with virtualenv, install dependencies, run python setup.py install in it and then execute whatever command you want it to do and report status. To do so I simply: - listed tests dependencies in test-requirements.txt (which are just nose and mock) - provide a tox.ini file which describe how to install the dependencies and execute nosetests - added the module 'coverage' to the list of test dependencies To run tests simply: pip install tox && tox That will execute the test command 'nosetests' using python2.6 and then python 2.7. The additional env 'cover' can be run using: tox -ecover. --- .gitignore | 3 +++ README.md | 13 ++++++++++++- test-requirements.txt | 4 ++++ tox.ini | 13 +++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 1a26c03a1..f6f85969d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ cover/ /doc/_build nbproject *.sublime-workspace + +/*egg-info +/.tox diff --git a/README.md b/README.md index 0b5f5a2a0..fcf74c3d6 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,9 @@ The object database implementation is optimized for handling large quantities of - Tested with nose 1.3.0 * Mock by Michael Foord used for tests - Tested with 1.0.1 +* Coverage - used for tests coverage -The list of dependencies are listed in /requirements.txt. The installer takes care of installing them for you though. +The list of dependencies are listed in /requirements.txt and /test-requirements.txt. The installer takes care of installing them for you though. ### INSTALL @@ -32,6 +33,16 @@ A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython +### RUNNING TESTS + +The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) a wrapper around virtualenv. It will take care of setting up environnements with the proper dependencies installed and execute test commands. To install it simply: + + pip install tox + +Then run: + + tox + ### DEVELOPMENT STATUS [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 000000000..6da60814c --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,4 @@ +# Remember to update README.md +coverage +nose +mock diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..a89b13482 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +[tox] +envlist = py26,py27 + +[testenv] +commands = nosetests +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + +[testenv:cover] +commands = nosetests --with-coverage + +[testenv:venv] +commands = {posargs} From d43055d44e58e8f010a71ec974c6a26f091a0b7a Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Fri, 25 Jul 2014 11:10:52 +0200 Subject: [PATCH 0295/3719] tox env to easily run flake8 Most people know about pep8 which enforce coding style. pyflakes goes a step beyond by analyzing the code. flake8 is basically a wrapper around both pep8 and pyflakes and comes with some additional checks. I find it very useful since you only need to require one package to have a lot of code issues reported to you. This patch provides a 'flake8' tox environement to easily install and run the utility on the code base. One simply has to: tox -eflake8 The env has been added to the default list of environement to have flake8 run by default. The repository in its current state does not pass checks but I noticed a pull request fixing pep8 issues. We can later easily ensure there is no regression by adjusting Travis configuration to run this env. More informations about flake8: https://pypi.python.org/pypi/flake8 --- test-requirements.txt | 1 + tox.ini | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6da60814c..116fbbd38 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ # Remember to update README.md coverage +flake8 nose mock diff --git a/tox.ini b/tox.ini index a89b13482..60bfb1d97 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27 +envlist = py26,py27,flake8 [testenv] commands = nosetests @@ -9,5 +9,12 @@ deps = -r{toxinidir}/requirements.txt [testenv:cover] commands = nosetests --with-coverage +[testenv:flake8] +commands = flake8 + [testenv:venv] commands = {posargs} + +[flake8] +#show-source = True +exclude = .tox,.venv,build,dist,doc,git/ext/ From fda32852a1dd6f6b0988248fb2c7921104e3ea6c Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Fri, 25 Jul 2014 21:12:33 +0200 Subject: [PATCH 0296/3719] warnings fixes --- setup.py | 2 ++ smmap/test/test_buf.py | 44 ++++++++++++----------- smmap/test/test_mman.py | 79 ++++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/setup.py b/setup.py index 13d2063ab..204a2a75d 100644 --- a/setup.py +++ b/setup.py @@ -52,4 +52,6 @@ "Programming Language :: Python :: 3.4", ], long_description=long_description, + tests_require=('nose', 'nosexcover'), + test_suite='nose.collector' ) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 15dfb8238..807d2770f 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,3 +1,5 @@ +from __future__ import with_statement, print_function + from .lib import TestBase, FileCreator from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager @@ -17,65 +19,66 @@ static_man = StaticWindowMapManager() class TestBuf(TestBase): - + def test_basics(self): fc = FileCreator(self.k_window_test_size, "buffer_test") - + # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - + buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None - + # can call end access any time buf.end_access() buf.end_access() assert len(buf) == 0 - + # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True assert len(buf) == fc.size - offset assert buf.cursor().is_valid() - + # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True assert len(buf) == fc.size assert buf.cursor().is_valid() - + # simple access - data = open(fc.path, 'rb').read() + with open(fc.path, 'rb') as fp: + data = fp.read() assert data[offset] == buf[0] assert data[offset:offset*2] == buf[0:offset] - + # negative indices, partial slices assert buf[-1] == buf[len(buf)-1] assert buf[-10:] == buf[len(buf)-10:len(buf)] - + # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() assert buf.cursor().is_associated() # but it remains associated - + # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically del(c) - + assert man_optimal.num_file_handles() == 1 - + # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to + # blast away with rnadom access and a full mapping - we don't want to # exagerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which + # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), + for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case'), (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) @@ -84,7 +87,7 @@ def test_basics(self): num_accesses_left = max_num_accesses num_bytes = 0 fsize = fc.size - + st = time() buf.begin_access() while num_accesses_left: @@ -102,7 +105,7 @@ def test_basics(self): num_bytes += 1 #END handle mode # END handle num accesses - + buf.end_access() assert manager.num_file_handles() assert manager.collect() @@ -110,8 +113,9 @@ def test_basics(self): elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed), + file=sys.stderr) # END handle access mode # END for each manager # END for each input diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e0516b21c..4d1839eca 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,3 +1,5 @@ +from __future__ import with_statement, print_function + from .lib import TestBase, FileCreator from smmap.mman import * @@ -12,43 +14,43 @@ from copy import copy class TestMMan(TestBase): - + def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") - + man = SlidingWindowMapManager() ci = WindowCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state - + cv = man.make_cursor(fc.path) assert not cv.is_valid() # no region mapped yet assert cv.is_associated()# but it know where to map it from assert cv.file_size() == fc.size assert cv.path() == fc.path - + # copy module cio = copy(cv) assert not cio.is_valid() and cio.is_associated() - + # assign method assert not ci.is_associated() ci.assign(cv) assert not ci.is_valid() and ci.is_associated() - + # unuse non-existing region is fine cv.unuse_region() cv.unuse_region() - + # destruction is fine (even multiple times) cv._destroy() WindowCursor(man)._destroy() - + def test_memory_manager(self): slide_man = SlidingWindowMapManager() static_man = StaticWindowMapManager() - + for man in (static_man, slide_man): assert man.num_file_handles() == 0 assert man.num_open_files() == 0 @@ -59,15 +61,15 @@ def test_memory_manager(self): assert man.window_size() > winsize_cmp_val assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 - + # collection doesn't raise in 'any' mode man._collect_lru_region(0) # doesn't raise if we are within the limit man._collect_lru_region(10) - - # doesn't fail if we overallocate + + # doesn't fail if we overallocate assert man._collect_lru_region(sys.maxsize) == 0 - + # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") fd = os.open(fc.path, os.O_RDONLY) @@ -77,8 +79,9 @@ def test_memory_manager(self): assert c.use_region(10, 10).is_valid() assert c.ofs_begin() == 10 assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] - + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + if isinstance(item, int): self.assertRaises(ValueError, c.path) else: @@ -87,38 +90,39 @@ def test_memory_manager(self): #END for each input os.close(fd) # END for each manager type - + def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") - data = open(fc.path, 'rb').read() + with open(fc.path, 'rb') as fp: + data = fp.read() fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 - #small_size = + #small_size = for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size - + # small windows, a reasonable max memory. Not too many regions at once man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) c = man.make_cursor(item) - + # still empty (more about that is tested in test_memory_manager() assert man.num_open_files() == 0 assert man.mapped_memory_size() == 0 - + base_offset = 5000 # window size is 0 for static managers, hence size will be 0. We take that into consideration size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() assert rr().client_count() == 2 # the manager and the cursor and us - + assert man.num_open_files() == 1 assert man.num_file_handles() == 1 assert man.mapped_memory_size() == rr().size() - + #assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded @@ -127,9 +131,9 @@ def test_memman_operation(self): else: assert rr().size() == fc.size #END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 assert c.use_region(0, nsize).is_valid() @@ -138,7 +142,7 @@ def test_memman_operation(self): assert c.size() == nsize assert c.ofs_begin() == 0 assert c.buffer()[:] == data[:nsize] - + # map some part at the end, our requested size cannot be kept overshoot = 4000 base_offset = fc.size - (size or c.size()) + overshoot @@ -156,23 +160,23 @@ def test_memman_operation(self): assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left assert rr().ofs_end() <= fc.size # it cannot be larger than the file assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - + # unising a region makes the cursor invalid c.unuse_region() assert not c.is_valid() if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only + # but doesn't change anything regarding the handle count - we cache it and only # remove mapped regions if we have to assert man.num_file_handles() == 2 #END ignore this for static managers - + # iterate through the windows, verify data contents # this will trigger map collection after a while max_random_accesses = 5000 num_random_accesses = max_random_accesses memory_read = 0 st = time() - + # cache everything to get some more performance includes_ofs = c.includes_ofs max_mapped_memory_size = man.max_mapped_memory_size() @@ -182,7 +186,7 @@ def test_memman_operation(self): while num_random_accesses: num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) - + # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() @@ -192,19 +196,20 @@ def test_memman_operation(self): csize = c.size() assert c.buffer()[:] == data[base_offset:base_offset+csize] memory_read += csize - + assert includes_ofs(base_offset) assert includes_ofs(base_offset+csize-1) assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) - sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed), + file=sys.stderr) + # an offset as large as the size doesn't work ! assert not c.use_region(fc.size, size).is_valid() - + # collection - it should be able to collect all assert man.num_file_handles() assert man.collect() From 3eb7265c532e99e8e434e31f910b05c055ae4369 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Fri, 25 Jul 2014 23:15:32 +0200 Subject: [PATCH 0297/3719] Ensure consistent output from git command The git command output can vary by language which would cause assertions errors when parsing the output. On POSIX system the language used by git can be adjusted by LC_MESSAGES. The special language 'C' is guaranteed to be always available and is whatever default the software has been written in (usually english, the case for git). Thus passing LC_MESSAGES to Popen will ensure we receive from git a consistent output regardless of the user preference. Addresses #153 --- git/cmd.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git/cmd.py b/git/cmd.py index b3274dd8f..a846fca86 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -316,6 +316,9 @@ def execute(self, command, if ouput_stream is True, the stdout value will be your output stream: * output_stream if extended_output = False * tuple(int(status), output_stream, str(stderr)) if extended_output = True + + Note git is executed with LC_MESSAGES="C" to ensure consitent + output regardless of system language. :raise GitCommandError: @@ -333,6 +336,7 @@ def execute(self, command, # Start the process proc = Popen(command, + env={"LC_MESSAGES": "C"}, cwd=cwd, stdin=istream, stderr=PIPE, From d45c76bd8cd28d05102311e9b4bc287819a51e0e Mon Sep 17 00:00:00 2001 From: Max Rasskazov Date: Mon, 8 Sep 2014 18:10:57 +0400 Subject: [PATCH 0298/3719] GPG signature support on commit object. Originals: Pull request "GPG signature support on commit object" #124 by Tatsuki Sugiura. https://github.com/gitpython-developers/GitPython/pull/124 commit 8065d2abdbb18e09560fc061807301b4c834d5a7 commit 62ecd6c66a84144632b045696326af503ee8cd4e --- git/objects/commit.py | 53 +++++++++++++++++++--------- git/test/fixtures/commit_with_gpgsig | 30 ++++++++++++++++ git/test/test_commit.py | 42 +++++++++++++++++++++- 3 files changed, 107 insertions(+), 18 deletions(-) create mode 100644 git/test/fixtures/commit_with_gpgsig diff --git a/git/objects/commit.py b/git/objects/commit.py index cbfd5097b..4380f4720 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -57,15 +57,15 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): __slots__ = ("tree", "author", "authored_date", "author_tz_offset", "committer", "committed_date", "committer_tz_offset", - "message", "parents", "encoding") + "message", "parents", "encoding", "gpgsig") _id_attribute_ = "binsha" def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, author_tz_offset=None, - committer=None, committed_date=None, committer_tz_offset=None, - message=None, parents=None, encoding=None): - """Instantiate a new Commit. All keyword arguments taking None as default will - be implicitly set on first query. - + committer=None, committed_date=None, committer_tz_offset=None, + message=None, parents=None, encoding=None, gpgsig=None): + """Instantiate a new Commit. All keyword arguments taking None as default will + be implicitly set on first query. + :param binsha: 20 byte sha1 :param parents: tuple( Commit, ... ) is a tuple of commit ids or actual Commits @@ -120,7 +120,8 @@ def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, aut self.parents = parents if encoding is not None: self.encoding = encoding - + self.gpgsig = gpgsig + @classmethod def _get_intermediate_items(cls, commit): return commit.parents @@ -393,7 +394,12 @@ def _serialize(self, stream): if self.encoding != self.default_encoding: write("encoding %s\n" % self.encoding) - + + if self.gpgsig: + write("gpgsig") + for sigline in self.gpgsig.rstrip("\n").split("\n"): + write(" "+sigline+"\n") + write("\n") # write plain bytes, be sure its encoded according to our encoding @@ -429,15 +435,28 @@ def _deserialize(self, stream): # now we can have the encoding line, or an empty line followed by the optional # message. self.encoding = self.default_encoding - # read encoding or empty line to separate message - enc = readline() - enc = enc.strip() - if enc: - self.encoding = enc[enc.find(' ')+1:] - # now comes the message separator - readline() - # END handle encoding - + + # read headers + buf = readline().strip() + while buf != "": + if buf[0:10] == "encoding ": + self.encoding = buf[buf.find(' ')+1:] + elif buf[0:7] == "gpgsig ": + sig = buf[buf.find(' ')+1:] + "\n" + is_next_header = False + while True: + sigbuf = readline() + if sigbuf == "": break + if sigbuf[0:1] != " ": + buf = sigbuf.strip() + is_next_header = True + break + sig += sigbuf[1:] + self.gpgsig = sig.rstrip("\n") + if is_next_header: + continue + buf = readline().strip() + # decode the authors name try: self.author.name = self.author.name.decode(self.encoding) diff --git a/git/test/fixtures/commit_with_gpgsig b/git/test/fixtures/commit_with_gpgsig new file mode 100644 index 000000000..f38cdabd6 --- /dev/null +++ b/git/test/fixtures/commit_with_gpgsig @@ -0,0 +1,30 @@ +tree cefbccb4843d821183ae195e70a17c9938318945 +parent 904435cf76a9bdd5eb41b1c4e049d5a64f3a8400 +author Jon Mason 1367013117 -0700 +committer Jon Mason 1368640702 -0700 +gpgsig -----BEGIN PGP SIGNATURE----- + Version: GnuPG v1.4.11 (GNU/Linux) + + iQIcBAABAgAGBQJRk8zMAAoJEG5mS6x6i9IjsTEP/0v2Wx/i7dqyKban6XMIhVdj + uI0DycfXqnCCZmejidzeao+P+cuK/ZAA/b9fU4MtwkDm2USvnIOrB00W0isxsrED + sdv6uJNa2ybGjxBolLrfQcWutxGXLZ1FGRhEvkPTLMHHvVriKoNFXcS7ewxP9MBf + NH97K2wauqA+J4BDLDHQJgADCOmLrGTAU+G1eAXHIschDqa6PZMH5nInetYZONDh + 3SkOOv8VKFIF7gu8X7HC+7+Y8k8U0TW0cjlQ2icinwCc+KFoG6GwXS7u/VqIo1Yp + Tack6sxIdK7NXJhV5gAeAOMJBGhO0fHl8UUr96vGEKwtxyZhWf8cuIPOWLk06jA0 + g9DpLqmy/pvyRfiPci+24YdYRBua/vta+yo/Lp85N7Hu/cpIh+q5WSLvUlv09Dmo + TTTG8Hf6s3lEej7W8z2xcNZoB6GwXd8buSDU8cu0I6mEO9sNtAuUOHp2dBvTA6cX + PuQW8jg3zofnx7CyNcd3KF3nh2z8mBcDLgh0Q84srZJCPRuxRcp9ylggvAG7iaNd + XMNvSK8IZtWLkx7k3A3QYt1cN4y1zdSHLR2S+BVCEJea1mvUE+jK5wiB9S4XNtKm + BX/otlTa8pNE3fWYBxURvfHnMY4i3HQT7Bc1QjImAhMnyo2vJk4ORBJIZ1FTNIhJ + JzJMZDRLQLFvnzqZuCjE + =przd + -----END PGP SIGNATURE----- + +NTB: Multiple NTB client fix + +Fix issue with adding multiple ntb client devices to the ntb virtual +bus. Previously, multiple devices would be added with the same name, +resulting in crashes. To get around this issue, add a unique number to +the device when it is added. + +Signed-off-by: Jon Mason diff --git a/git/test/test_commit.py b/git/test/test_commit.py index 58e511517..f536470f5 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -13,6 +13,7 @@ from cStringIO import StringIO import time import sys +import re def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False): @@ -272,4 +273,43 @@ def test_serialization_unicode_support(self): # actually, it can't be printed in a shell as repr wants to have ascii only # it appears cmt.author.__repr__() - + + def test_gpgsig(self): + cmt = self.rorepo.commit() + cmt._deserialize(open(fixture_path('commit_with_gpgsig'))) + + fixture_sig = """-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.11 (GNU/Linux) + +iQIcBAABAgAGBQJRk8zMAAoJEG5mS6x6i9IjsTEP/0v2Wx/i7dqyKban6XMIhVdj +uI0DycfXqnCCZmejidzeao+P+cuK/ZAA/b9fU4MtwkDm2USvnIOrB00W0isxsrED +sdv6uJNa2ybGjxBolLrfQcWutxGXLZ1FGRhEvkPTLMHHvVriKoNFXcS7ewxP9MBf +NH97K2wauqA+J4BDLDHQJgADCOmLrGTAU+G1eAXHIschDqa6PZMH5nInetYZONDh +3SkOOv8VKFIF7gu8X7HC+7+Y8k8U0TW0cjlQ2icinwCc+KFoG6GwXS7u/VqIo1Yp +Tack6sxIdK7NXJhV5gAeAOMJBGhO0fHl8UUr96vGEKwtxyZhWf8cuIPOWLk06jA0 +g9DpLqmy/pvyRfiPci+24YdYRBua/vta+yo/Lp85N7Hu/cpIh+q5WSLvUlv09Dmo +TTTG8Hf6s3lEej7W8z2xcNZoB6GwXd8buSDU8cu0I6mEO9sNtAuUOHp2dBvTA6cX +PuQW8jg3zofnx7CyNcd3KF3nh2z8mBcDLgh0Q84srZJCPRuxRcp9ylggvAG7iaNd +XMNvSK8IZtWLkx7k3A3QYt1cN4y1zdSHLR2S+BVCEJea1mvUE+jK5wiB9S4XNtKm +BX/otlTa8pNE3fWYBxURvfHnMY4i3HQT7Bc1QjImAhMnyo2vJk4ORBJIZ1FTNIhJ +JzJMZDRLQLFvnzqZuCjE +=przd +-----END PGP SIGNATURE-----""" + assert cmt.gpgsig == fixture_sig + + cmt.gpgsig = "" + assert cmt.gpgsig != fixture_sig + + cstream = StringIO() + cmt._serialize(cstream) + assert re.search(r"^gpgsig $", cstream.getvalue(), re.MULTILINE) + + cstream.seek(0) + cmt.gpgsig = None + cmt._deserialize(cstream) + assert cmt.gpgsig == "" + + cmt.gpgsig = None + cstream = StringIO() + cmt._serialize(cstream) + assert not re.search(r"^gpgsig ", cstream.getvalue(), re.MULTILINE) From cc32b0a99744b07bcacf3b027af597a4bdd10b85 Mon Sep 17 00:00:00 2001 From: Orgad Shaneh Date: Wed, 10 Sep 2014 09:51:54 +0300 Subject: [PATCH 0299/3719] Setup: Fix invalid syntax --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62bc6d007..07f810be8 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def run(self): try: build_ext.run(self) except Exception: - print "Ignored failure when building extensions, pure python modules will be used instead" + print("Ignored failure when building extensions, pure python modules will be used instead") # END ignore errors From 6c9fcd7745d2f0c933b46a694f77f85056133ca5 Mon Sep 17 00:00:00 2001 From: Jan Vcelak Date: Wed, 21 Mar 2012 17:45:04 +0100 Subject: [PATCH 0300/3719] Fix issue #41: repo.is_dirty() on empty repository with stashed files --- git/repo/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 71492fe87..2296cf295 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -499,8 +499,8 @@ def is_dirty(self, index=True, working_tree=True, untracked_files=False): default_args = ('--abbrev=40', '--full-index', '--raw') if index: # diff index against HEAD - if isfile(self.index.path) and self.head.is_valid() and \ - len(self.git.diff('HEAD', '--cached', *default_args)): + if isfile(self.index.path) and \ + len(self.git.diff('--cached', *default_args)): return True # END index handling if working_tree: From 6a61110053bca2bbf605b66418b9670cbd555802 Mon Sep 17 00:00:00 2001 From: "Marcus R. Brown" Date: Fri, 11 Jan 2013 00:25:28 -0700 Subject: [PATCH 0301/3719] Fix the `git version` parser. --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index b3274dd8f..18f7c714a 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -236,7 +236,7 @@ def _set_cache_(self, attr): if attr == '_version_info': # We only use the first 4 numbers, as everthing else could be strings in fact (on windows) version_numbers = self._call_process('version').split(' ')[2] - self._version_info = tuple(int(n) for n in version_numbers.split('.')[:4]) + self._version_info = tuple(int(n) for n in version_numbers.split('.')[:4] if n.isdigit()) else: super(Git, self)._set_cache_(attr) #END handle version info From a9f7f7bc5a2819924992dcb536c8e72399ae6f16 Mon Sep 17 00:00:00 2001 From: Matt Hickford Date: Mon, 6 Oct 2014 17:35:19 +0100 Subject: [PATCH 0302/3719] Add installation instructions to readme Also, fix broken badge link. --- README.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0fc0534e6..f97d5c31e 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,20 @@ aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to operate on large objects with a small memory footprint. +Installation +============ + +.. image:: https://pypip.in/version/gitdb/badge.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Latest Version +.. image:: https://pypip.in/py_versions/gitdb/badge.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Supported Python versions + +From `PyPI `_ + + pip install gitdb + REQUIREMENTS ============ @@ -33,8 +47,9 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - +.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master + :target: https://travis-ci.org/gitpython-developers/gitdb + https://github.com/gitpython-developers/gitdb/issues LICENSE From 0e0b7e253f7635af9e24a15f4393481495565667 Mon Sep 17 00:00:00 2001 From: Matt Hickford Date: Mon, 6 Oct 2014 17:39:18 +0100 Subject: [PATCH 0303/3719] Clarify which Python versions are supported --- setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 07f810be8..6dd4fa4b4 100755 --- a/setup.py +++ b/setup.py @@ -90,5 +90,13 @@ def get_data_files(self): zip_safe=False, requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), - long_description = """GitDB is a pure-Python git object database""" + long_description = """GitDB is a pure-Python git object database""", + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], ) From ef59e98f74eeb00373b02f7a2b8bed1943924de9 Mon Sep 17 00:00:00 2001 From: David Schryer Date: Wed, 5 Nov 2014 10:48:33 +0200 Subject: [PATCH 0304/3719] Minor change to begin Python3 compatibility support. --- gitdb/db/ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 60004a77a..0a28b9efb 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -68,7 +68,7 @@ def _update_dbs_from_ref_file(self): db.databases() # END verification self._dbs.append(db) - except Exception, e: + except Exception as e: # ignore invalid paths or issues pass # END for each path to add From 48f5476867d8316ee1af55e0e7cfacacbdf0ad68 Mon Sep 17 00:00:00 2001 From: Tamas Pal Date: Wed, 5 Nov 2014 17:13:31 +0100 Subject: [PATCH 0305/3719] GitRunCommand exception can store stdout output too. Some git commands, like git merge outputs their problems onto stdout, instead of stderr, which will be thrown away by the current setup. This change allows the GitPython commands to store the stdout's value too, in case of error. --- git/cmd.py | 5 ++++- git/exc.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index b3274dd8f..5323a63c5 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -380,7 +380,10 @@ def execute(self, command, # END handle debug printing if with_exceptions and status != 0: - raise GitCommandError(command, status, stderr_value) + if with_extended_output: + raise GitCommandError(command, status, stderr_value, stdout_value) + else: + raise GitCommandError(command, status, stderr_value) # Allow access to the command's status code if with_extended_output: diff --git a/git/exc.py b/git/exc.py index 3b3091e2a..76d3d4865 100644 --- a/git/exc.py +++ b/git/exc.py @@ -17,14 +17,18 @@ class NoSuchPathError(OSError): class GitCommandError(Exception): """ Thrown if execution of the git command fails with non-zero status code. """ - def __init__(self, command, status, stderr=None): + def __init__(self, command, status, stderr=None, stdout=None): self.stderr = stderr + self.stdout = stdout self.status = status self.command = command def __str__(self): - return ("'%s' returned exit status %i: %s" % - (' '.join(str(i) for i in self.command), self.status, self.stderr)) + ret = "'%s' returned exit status %i: %s" % \ + (' '.join(str(i) for i in self.command), self.status, self.stderr) + if self.stdout is not None: + ret += "\nstdout: %s" % self.stdout + return ret class CheckoutError( Exception ): From f03e6162f99e4bfdd60c08168dabef3a1bdb1825 Mon Sep 17 00:00:00 2001 From: Craig Northway Date: Fri, 25 Jul 2014 11:53:57 +1000 Subject: [PATCH 0306/3719] Basic test for __unpack_args to verify unicode handling works (cherry picked from commit 8fa25b1cd5a82679c7b12d546b96c30cafed0559) Signed-off-by: David Black Conflicts: git/test/test_git.py --- git/test/test_git.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/git/test/test_git.py b/git/test/test_git.py index e67cb92b0..5d4756baf 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -5,8 +5,9 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os, sys -from git.test.lib import ( TestBase, - patch, +from git.test.lib import ( + TestBase, + patch, raises, assert_equal, assert_true, @@ -16,7 +17,7 @@ GitCommandError ) class TestGit(TestBase): - + @classmethod def setUp(cls): super(TestGit, cls).setUp() @@ -29,6 +30,14 @@ def test_call_process_calls_execute(self, git): assert_true(git.called) assert_equal(git.call_args, ((['git', 'version'],), {})) + def test_call_unpack_args_unicode(self): + args = Git._Git__unpack_args(u'Unicode' + unichr(40960)) + assert_equal(args, ['Unicode\xea\x80\x80']) + + def test_call_unpack_args(self): + args = Git._Git__unpack_args(['git', 'log', '--', u'Unicode' + unichr(40960)]) + assert_equal(args, ['git', 'log', '--', 'Unicode\xea\x80\x80']) + @raises(GitCommandError) def test_it_raises_errors(self): self.git.this_does_not_exist() @@ -58,7 +67,7 @@ def test_it_ignores_false_kwargs(self, git): # this_should_not_be_ignored=False implies it *should* be ignored output = self.git.version(pass_this_kwarg=False) assert_true("pass_this_kwarg" not in git.call_args[1]) - + def test_persistent_cat_file_command(self): # read header only import subprocess as sp @@ -67,37 +76,37 @@ def test_persistent_cat_file_command(self): g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info = g.stdout.readline() - + # read header + data g = self.git.cat_file(batch=True, istream=sp.PIPE,as_process=True) g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info_two = g.stdout.readline() assert obj_info == obj_info_two - + # read data - have to read it in one large chunk size = int(obj_info.split()[2]) data = g.stdout.read(size) terminating_newline = g.stdout.read(1) - + # now we should be able to read a new object g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() assert g.stdout.readline() == obj_info - - + + # same can be achived using the respective command functions hexsha, typename, size = self.git.get_object_header(hexsha) hexsha, typename_two, size_two, data = self.git.get_object_data(hexsha) assert typename == typename_two and size == size_two - + def test_version(self): v = self.git.version_info assert isinstance(v, tuple) for n in v: assert isinstance(n, int) #END verify number types - + def test_cmd_override(self): prev_cmd = self.git.GIT_PYTHON_GIT_EXECUTABLE try: From eb52c96d7e849e68fda40e4fa7908434e7b0b022 Mon Sep 17 00:00:00 2001 From: Craig Northway Date: Fri, 18 Jul 2014 08:35:59 +1000 Subject: [PATCH 0307/3719] Fixing unicode types (cherry picked from commit ca2b901e7229fc5c793762fd4e4c1c38c5a78e80) --- git/cmd.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git/cmd.py b/git/cmd.py index b3274dd8f..73126fba6 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -410,12 +410,16 @@ def transform_kwargs(self, split_single_char_options=False, **kwargs): @classmethod def __unpack_args(cls, arg_list): if not isinstance(arg_list, (list,tuple)): + if isinstance(arg_list, unicode): + return [arg_list.encode('utf-8')] return [ str(arg_list) ] outlist = list() for arg in arg_list: if isinstance(arg_list, (list, tuple)): outlist.extend(cls.__unpack_args( arg )) + elif isinstance(arg_list, unicode): + outlist.append(arg_list.encode('utf-8')) # END recursion else: outlist.append(str(arg)) From e8987f2746637cbe518e6fe5cf574a9f151472ed Mon Sep 17 00:00:00 2001 From: David Black Date: Wed, 12 Nov 2014 13:39:18 +1100 Subject: [PATCH 0308/3719] Switch http://github.com/gitpython-developers/gitdb.git to https://github.com/gitpython-developers/gitdb.git . Signed-off-by: David Black --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 533fc59f2..612c39d95 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "gitdb"] path = git/ext/gitdb - url = http://github.com/gitpython-developers/gitdb.git + url = https://github.com/gitpython-developers/gitdb.git From c390e223553964fc8577d6837caf19037c4cd6f6 Mon Sep 17 00:00:00 2001 From: David Black Date: Wed, 12 Nov 2014 15:50:15 +1100 Subject: [PATCH 0309/3719] Fix the Repo commit and tree methods to work with unicode revs. Signed-off-by: David Black --- git/repo/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 8191b3057..a45d215ea 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -376,7 +376,7 @@ def commit(self, rev=None): if rev is None: return self.head.commit else: - return self.rev_parse(str(rev)+"^0") + return self.rev_parse(unicode(rev)+"^0") def iter_trees(self, *args, **kwargs): """:return: Iterator yielding Tree objects @@ -399,7 +399,7 @@ def tree(self, rev=None): if rev is None: return self.head.commit.tree else: - return self.rev_parse(str(rev)+"^{tree}") + return self.rev_parse(unicode(rev)+"^{tree}") def iter_commits(self, rev=None, paths='', **kwargs): """A list of Commit objects representing the history of a given ref/commit From 977e666e2489ddc669a06481bb5192b59854da8d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 12 Nov 2014 20:30:47 +0100 Subject: [PATCH 0310/3719] Initial improvements to get rid of the performance regression in py3. Byte buffer concatenations are considerably slower here for some reason. Also there was no need for the memorybuffer. --- .travis.yml | 2 ++ README.md | 7 +------ doc/source/changes.rst | 6 +++++- smmap/buf.py | 10 +++++----- smmap/mman.py | 29 ++++++++++++++--------------- smmap/util.py | 5 +++-- 6 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47cb41170..1f7872c86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python python: + - 2.4 + - 2.5 - 2.6 - 2.7 - 3.3 diff --git a/README.md b/README.md index c056ed1c1..6d2353820 100644 --- a/README.md +++ b/README.md @@ -25,19 +25,15 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.4, 2.5 or 2.6 +* Python 2.4, 2.5, 2.6, 2.7 or 3.3 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. - - ## Limitations * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. -* It wasn't tested on python 2.7 and 3.x. - ## Installing smmap @@ -80,7 +76,6 @@ Issues can be filed on github: * https://github.com/Byron/smmap/issues - ## License Information *smmap* is licensed under the New BSD License. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 03148fb31..6174008e9 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,12 +2,16 @@ Changelog ######### +********** +v0.8.2 +********** +- Cleaned up code and assured it works sufficiently well with python 3 + ********** v0.8.1 ********** - A single bugfix - ********** v0.8.0 ********** diff --git a/smmap/buf.py b/smmap/buf.py index 2f27d4d01..4cde6dde4 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -79,18 +79,18 @@ def __getslice__(self, i, j): else: l = j-i # total length ofs = i - # Keeping tokens in a list could possible be faster, but the list - # overhead outweighs the benefits (tested) ! - md = bytes() + # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower + # in the previous iteration of this code + md = list() while l: c.use_region(ofs, l) assert c.is_valid() d = c.buffer()[:l] ofs += len(d) l -= len(d) - md += d + md.append(d) #END while there are bytes to read - return md + return bytes().join(md) # END fast or slow path #{ Interface diff --git a/smmap/mman.py b/smmap/mman.py index 7cbb535bd..a89efd474 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,13 +1,12 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - align_to_mmap, - string_types, - buffer, - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + string_types, + buffer, + ) from weakref import ref import sys @@ -261,14 +260,14 @@ class StaticWindowMapManager(object): def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): """initialize the manager with the given parameters. :param window_size: if -1, a default window size will be chosen depending on - the operating system's architechture. It will internally be quantified to a multiple of the page size + the operating system's architecture. It will internally be quantified to a multiple of the page size If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. - If 0, a viable default iwll be set dependning on the system's architecture. + If 0, a viable default will be set depending on the system's architecture. It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate - :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, - the manager will free as many handles as posisble""" + :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + the manager will free as many handles as possible""" self._fdict = dict() self._window_size = window_size self._max_memory_size = max_memory_size @@ -277,7 +276,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._handle_count = 0 if window_size < 0: - coeff = 32 + coeff = 64 if is_64_bit(): coeff = 1024 #END handle arch @@ -285,7 +284,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. # END handle max window size if max_memory_size == 0: - coeff = 512 + coeff = 1024 if is_64_bit(): coeff = 8192 #END handle arch diff --git a/smmap/util.py b/smmap/util.py index c37dfdd31..3396d90c0 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,8 +23,9 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - return memoryview(obj[offset:offset+size]) - + # return memoryview(obj[offset:offset+size]) + # doing it directly is much faster ! + return obj[offset:offset+size] def string_types(): if sys.version_info[0] >= 3: From 948a9274527d14702875581d7115389cf9aa8244 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 08:21:09 +0100 Subject: [PATCH 0311/3719] Fixed a few typos and major linter errors --- .travis.yml | 5 +++-- README.md | 4 +--- doc/source/changes.rst | 2 +- doc/source/intro.rst | 8 ++------ setup.py | 10 +++++----- smmap/__init__.py | 2 +- smmap/buf.py | 2 -- smmap/mman.py | 16 ++++++---------- smmap/test/test_buf.py | 14 +++++++++----- smmap/test/test_mman.py | 12 +++++++----- smmap/test/test_util.py | 9 ++++++++- smmap/util.py | 4 ++-- 12 files changed, 45 insertions(+), 43 deletions(-) mode change 100644 => 100755 setup.py diff --git a/.travis.yml b/.travis.yml index 1f7872c86..cb0c16e44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: python python: - - 2.4 - - 2.5 + # These versions are unsupported by travis, even though smmap claims to still support these outdated versions + # - 2.4 + # - 2.5 - 2.6 - 2.7 - 3.3 diff --git a/README.md b/README.md index 6d2353820..327d66372 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,9 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively: +Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: ```bash -$ easy_install smmap -# or $ pip install smmap ``` diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6174008e9..d5ed8e378 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ********** -v0.8.2 +v0.8.3 ********** - Cleaned up code and assured it works sufficiently well with python 3 diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 30bff0ded..ee3108a6a 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.4, 2.5 or 2.6 +* Python 2.4, 2.5, 2.6, 2.7 or 3.3 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. @@ -32,15 +32,12 @@ Limitations ########### * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. -* It wasn't tested on python 2.7 and 3.x. ################ Installing smmap ################ -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: +Its easiest to install smmap using the *pip* program:: - $ easy_install smmap - # or $ pip install smmap As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. @@ -75,5 +72,4 @@ License Information ################### *smmap* is licensed under the New BSD License. -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools .. _pip: http://www.pip-installer.org/en/latest/ diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 204a2a75d..c6afc8960 --- a/setup.py +++ b/setup.py @@ -10,10 +10,10 @@ import smmap -if os.path.exists("README.rst"): - long_description = codecs.open('README.rst', "r", "utf-8").read() +if os.path.exists("README.md"): + long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/nvie/smmap/tree/master" + long_description = "See http://github.com/Byron/smmap" setup( name="smmap", @@ -32,8 +32,8 @@ #"Development Status :: 1 - Planning", #"Development Status :: 2 - Pre-Alpha", #"Development Status :: 3 - Alpha", - "Development Status :: 4 - Beta", - #"Development Status :: 5 - Production/Stable", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", #"Development Status :: 6 - Mature", #"Development Status :: 7 - Inactive", "Environment :: Console", diff --git a/smmap/__init__.py b/smmap/__init__.py index 879ebea24..c494648d7 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 2) +version_info = (0, 8, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index 4cde6dde4..ef9d49e46 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,6 +1,4 @@ """Module with a simple buffer implementation using the memory manager""" -from .mman import WindowCursor - import sys __all__ = ["SlidingWindowMapBuffer"] diff --git a/smmap/mman.py b/smmap/mman.py index a89efd474..da6fd8153 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -101,7 +101,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. - This is not the case if the mapping failed becaues we reached the end of the file + This is not the case if the mapping failed because we reached the end of the file **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" @@ -137,7 +137,7 @@ def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it + to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None # note: should reset ofs and size, but we spare that for performance. Its not @@ -203,7 +203,7 @@ def file_size(self): return self._rlist.file_size() def path_or_fd(self): - """:return: path or file decriptor of the underlying mapped file""" + """:return: path or file descriptor of the underlying mapped file""" return self._rlist.path_or_fd() def path(self): @@ -237,12 +237,12 @@ class StaticWindowMapManager(object): These clients would have to use a SlidingWindowMapBuffer to hide this fact. This type will always use a maximum window size, and optimize certain methods to - acomodate this fact""" + accommodate this fact""" __slots__ = [ '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_memory_size', # maximum amount of memory we may allocate '_max_handle_count', # maximum amount of handles to keep open '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles @@ -264,7 +264,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default will be set depending on the system's architecture. - It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate + It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, the manager will free as many handles as possible""" @@ -350,8 +350,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # As many more operations are likely to fail in that condition ( # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway if is_recursive: # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it @@ -562,8 +560,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # As many more operations are likely to fail in that condition ( # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway if is_recursive: # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 807d2770f..d3e51e2ee 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,14 +1,18 @@ -from __future__ import with_statement, print_function +from __future__ import print_function from .lib import TestBase, FileCreator -from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager -from smmap.buf import * +from smmap.mman import ( + SlidingWindowMapManager, + StaticWindowMapManager + ) +from smmap.buf import SlidingWindowMapBuffer from random import randint from time import time import sys import os +import logging man_optimal = SlidingWindowMapManager() @@ -71,8 +75,8 @@ def test_basics(self): assert man_optimal.num_file_handles() == 1 # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to - # exagerate the manager's overhead, but measure the buffer overhead + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 4d1839eca..cc5d91488 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,11 +1,13 @@ -from __future__ import with_statement, print_function +from __future__ import print_function from .lib import TestBase, FileCreator -from smmap.mman import * -from smmap.mman import WindowCursor +from smmap.mman import ( + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager + ) from smmap.util import align_to_mmap -from smmap.exc import RegionCollectionError from random import randint from time import time @@ -67,7 +69,7 @@ def test_memory_manager(self): # doesn't raise if we are within the limit man._collect_lru_region(10) - # doesn't fail if we overallocate + # doesn't fail if we over-allocate assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 8afba005e..745da83d7 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,6 +1,13 @@ from .lib import TestBase, FileCreator -from smmap.util import * +from smmap.util import ( + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap + ) import os import sys diff --git a/smmap/util.py b/smmap/util.py index 3396d90c0..a4d7d8f1d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -113,7 +113,7 @@ class MapRegion(object): '__weakref__' ] _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 - + if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot @@ -283,4 +283,4 @@ def file_size(self): #END update file size return self._file_size -#} END utilty classes +#} END utility classes From 85dde34bc724570617f3df1cdc40ba1b0942c77e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 09:00:44 +0100 Subject: [PATCH 0312/3719] Minor adjustments to adapt to changes in async (due to be removed anyway) --- gitdb/__init__.py | 2 +- gitdb/base.py | 6 +----- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 2 +- gitdb/pack.py | 3 ++- gitdb/stream.py | 3 +-- gitdb/test/lib.py | 2 -- gitdb/test/test_stream.py | 3 +-- gitdb/util.py | 23 +++++++++-------------- setup.py | 35 ++++++++++++++++++++++++++--------- 11 files changed, 44 insertions(+), 39 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ff750d14c..bb1d13b47 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 4) +version_info = (0, 5, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/base.py b/gitdb/base.py index bad5f7472..3476d0d1a 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -3,11 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" -from util import ( - bin_to_hex, - zlib - ) - +from util import bin_to_hex from fun import ( type_id_to_type_map, type_to_type_id_map diff --git a/gitdb/ext/async b/gitdb/ext/async index 90326fb86..b930ee15c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 +Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 616e9ceaf..f53ddc686 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd +Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf diff --git a/gitdb/fun.py b/gitdb/fun.py index c1e73e895..177a2fb56 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -10,7 +10,7 @@ BadObjectType ) -from util import zlib +import zlib decompressobj = zlib.decompressobj import mmap diff --git a/gitdb/pack.py b/gitdb/pack.py index 48121f026..e2673ee1b 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -3,13 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" +import zlib + from gitdb.exc import ( BadObject, UnsupportedOperation, ParseError ) from util import ( - zlib, mman, LazyMixin, unpack_from, diff --git a/gitdb/stream.py b/gitdb/stream.py index 6441b1e1a..cbb10c1f1 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from cStringIO import StringIO -import errno +import zlib import mmap import os @@ -23,7 +23,6 @@ make_sha, write, close, - zlib ) has_perf_mod = False diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ac8473a4e..af57e46d1 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -11,8 +11,6 @@ ZippedStoreShaWriter ) -from gitdb.util import zlib - import sys import random from array import array diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 6dc27463c..d2487f6d1 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -18,12 +18,11 @@ hex_to_bin ) -from gitdb.util import zlib +import zlib from gitdb.typ import ( str_blob_type ) -import time import tempfile import os diff --git a/gitdb/util.py b/gitdb/util.py index 1662b662d..75f6bb927 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -7,27 +7,22 @@ import mmap import sys import errno - -from cStringIO import StringIO +import stat # in py 2.4, StringIO is only StringI, without write support. # Hence we must use the python implementation for this if sys.version_info[1] < 5: from StringIO import StringIO +else: + from cStringIO import StringIO # END handle python 2.4 -try: - import async.mod.zlib as zlib -except ImportError: - import zlib -# END try async zlib - from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer + ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -304,7 +299,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode, 0600) + fd = os.open(self._lockfilepath(), lockmode, stat.S_IREAD|stat.S_IWRITE) if not write: os.close(fd) else: @@ -373,7 +368,7 @@ def _end_writing(self, successful=True): # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well - chmod(self._filepath, 0644) + chmod(self._filepath, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) else: # just delete the file so far, we failed os.remove(lockfile) diff --git a/setup.py b/setup.py index 6dd4fa4b4..639370471 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 4) +version_info = (0, 5, 5) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, @@ -88,15 +88,32 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), + requires=('async (>=0.6.2)', 'smmap (>=0.8.3)'), install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - ], - ) + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + ],) From a8f2f63823324ad76cbb36b0f4115e73c7d9d594 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 10:31:45 +0100 Subject: [PATCH 0313/3719] Made sure xrange is used instead of range in python 2 range in py2 will return a list, which can mean a lot of time and memory is spent on generating it even though it's just used for iteration. Simplified implementation of MAXSIZE --- gitdb/db/pack.py | 3 ++- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/pack.py | 10 +++++----- gitdb/test/db/lib.py | 5 +++-- gitdb/test/lib.py | 3 ++- gitdb/test/test_pack.py | 3 ++- gitdb/util.py | 5 +---- gitdb/utils/compat.py | 20 +++++--------------- 9 files changed, 22 insertions(+), 31 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eca02bbff..b95bfed9d 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -18,6 +18,7 @@ ) from gitdb.pack import PackEntity +from gitdb.utils.compat import xrange from functools import reduce @@ -106,7 +107,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in range(index.size()): + for index in xrange(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/ext/async b/gitdb/ext/async index 3f26b05c2..b930ee15c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 3f26b05c2f1a079d5807ed15c01b053ee846e745 +Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 552671191..f53ddc686 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 55267119140f3828a24b4986600ed21a1808d6cc +Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf diff --git a/gitdb/pack.py b/gitdb/pack.py index 4e83ba39d..0b3ffd53e 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -64,7 +64,7 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import izip, buffer +from gitdb.utils.compat import izip, buffer, xrange import tempfile import array @@ -208,7 +208,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in range(255): + for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i+1] += v @@ -374,7 +374,7 @@ def _read_fanout(self, byte_offset): d = self._cursor.map() out = list() append = out.append - for i in range(256): + for i in xrange(256): append(unpack_from('>L', d, byte_offset + i*4)[0]) # END for each entry return out @@ -415,7 +415,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in range(self.size())) + return tuple(self.offset(index) for index in xrange(self.size())) # END handle version def sha_to_index(self, sha): @@ -703,7 +703,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in range(self._index.size()): + for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 38747deb2..8e333ddf2 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -22,6 +22,7 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type from gitdb.utils.encoding import force_bytes +from gitdb.utils.compat import xrange from async import IteratorReader @@ -43,7 +44,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in range(ni): + for i in xrange(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) @@ -131,7 +132,7 @@ def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 def istream_generator(offset=0, ni=ni): - for data_src in range(ni): + for data_src in xrange(ni): data = bytes(data_src + offset) yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 75342f173..d692224ff 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -12,6 +12,7 @@ ) from gitdb.util import zlib +from gitdb.utils.compat import xrange import sys import random @@ -110,7 +111,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes // 4 - producer = range(actual_size) + producer = xrange(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index bcda3cfb8..6b67ad887 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -26,6 +26,7 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha +from gitdb.utils.compat import xrange from itertools import chain try: @@ -64,7 +65,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in range(index.size()): + for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/util.py b/gitdb/util.py index 5a82c553e..a3c44d446 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -122,13 +122,10 @@ def byte_ord(b): #{ Routines -def make_sha(source=None): +def make_sha(source=''.encode("ascii")): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ - if source is None: - source = "".encode("ascii") - try: return hashlib.sha1(source) except NameError: diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index b9da683fa..eec319f3d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,8 +4,10 @@ try: from itertools import izip + xrange = xrange except ImportError: izip = zip + xrange = range try: # Python 2 @@ -21,19 +23,7 @@ def buffer(obj, offset, size=None): memoryview = memoryview -if PY3: +try: + MAXSIZE = sys.maxint +except AttributeError: MAXSIZE = sys.maxsize -else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X From 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 11:28:27 +0100 Subject: [PATCH 0314/3719] Fixed incorrect usage of memoryview. It's not getting faster though [ skip ci ] --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index a4d7d8f1d..44e94124d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,7 +23,7 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - # return memoryview(obj[offset:offset+size]) + # return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! return obj[offset:offset+size] From b64c771bcb2ec336dd549cfe9d072340c886f3c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 12:17:01 +0100 Subject: [PATCH 0315/3719] Fixed all applicable lint issues --- gitdb/db/base.py | 2 -- gitdb/db/git.py | 7 +------ gitdb/db/loose.py | 8 -------- gitdb/db/pack.py | 1 - gitdb/db/ref.py | 1 - gitdb/ext/smmap | 2 +- gitdb/fun.py | 11 +++-------- gitdb/pack.py | 7 ++----- gitdb/stream.py | 4 +--- gitdb/test/db/lib.py | 8 ++++++-- gitdb/test/db/test_git.py | 6 +++++- gitdb/test/db/test_loose.py | 5 ++++- gitdb/test/db/test_mem.py | 5 ++++- gitdb/test/db/test_pack.py | 11 +++++++---- gitdb/test/lib.py | 9 +-------- gitdb/test/performance/lib.py | 4 +--- gitdb/test/performance/test_pack.py | 2 +- gitdb/test/performance/test_pack_streaming.py | 1 - gitdb/test/performance/test_stream.py | 18 +++++++----------- gitdb/test/test_base.py | 10 +++++++++- gitdb/test/test_example.py | 5 ++++- gitdb/test/test_pack.py | 3 --- gitdb/test/test_stream.py | 11 ++++++----- gitdb/test/test_util.py | 1 - gitdb/util.py | 6 +++++- gitdb/utils/compat.py | 10 ++++++++-- 26 files changed, 76 insertions(+), 82 deletions(-) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 53a94d249..c534705b7 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,7 +22,6 @@ from itertools import chain from functools import reduce -import sys __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -206,7 +205,6 @@ def _databases_recursive(database, output): """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" if isinstance(database, CompoundDB): - compounds = list() dbs = database.databases() output.extend(db for db in dbs if not isinstance(db, CompoundDB)) for cdb in (db for db in dbs if isinstance(db, CompoundDB)): diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 5c74a2049..d22e3f1b9 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -12,12 +12,7 @@ from gitdb.db.pack import PackedDB from gitdb.db.ref import ReferenceDB -from gitdb.util import LazyMixin -from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName -) +from gitdb.exc import InvalidDBRoot import os diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 63f96352b..3abdaa96f 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -10,7 +10,6 @@ from gitdb.exc import ( - InvalidDBRoot, BadObject, AmbiguousObjectName ) @@ -55,8 +54,6 @@ from gitdb.utils.encoding import force_bytes import tempfile -import mmap -import sys import os @@ -149,11 +146,6 @@ def _map_loose_object(self, sha): raise BadObject(sha) # END handle error # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index b95bfed9d..4d0a7f8b3 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -103,7 +103,6 @@ def stream(self, sha): return entity.stream_at_index(index) def sha_iter(self): - sha_list = list() for entity in self.entities(): index = entity.index() sha_by_index = index.sha diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 748f7c145..d98912643 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -6,7 +6,6 @@ CompoundDB, ) -import os __all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f53ddc686..28fd45e0a 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf +Subproject commit 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 diff --git a/gitdb/fun.py b/gitdb/fun.py index 22e56ddbb..80f01e6b2 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -6,18 +6,15 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from gitdb.exc import ( - BadObjectType -) - import zlib from gitdb.util import byte_ord decompressobj = zlib.decompressobj import mmap from itertools import islice +from functools import reduce -from gitdb.utils.compat import izip +from gitdb.utils.compat import izip, buffer, xrange from gitdb.typ import ( str_blob_type, str_commit_type, @@ -247,7 +244,6 @@ def compress(self): if slen < 2: return self i = 0 - slen_orig = slen first_data_index = None while i < slen: @@ -399,7 +395,6 @@ def loose_object_header_info(m): object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" from gitdb.const import NULL_BYTE - from gitdb.utils.encoding import force_text decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) @@ -684,7 +679,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - # raise ImportError; # DEBUG + # NOQA from _perf import connect_deltas except ImportError: pass diff --git a/gitdb/pack.py b/gitdb/pack.py index fbfd18b08..87818b6e7 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -7,6 +7,7 @@ from gitdb.exc import ( BadObject, + AmbiguousObjectName, UnsupportedOperation, ParseError ) @@ -57,11 +58,7 @@ FlexibleSha1Writer ) -from struct import ( - pack, - unpack, -) - +from struct import pack from binascii import crc32 from gitdb.const import NULL_BYTE diff --git a/gitdb/stream.py b/gitdb/stream.py index de5884859..5daf01cb6 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,9 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from io import BytesIO, StringIO +from io import BytesIO -import errno import mmap import os import zlib @@ -15,7 +14,6 @@ stream_copy, apply_delta_data, connect_deltas, - DeltaChunkList, delta_types ) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 8e333ddf2..67958c366 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -6,12 +6,16 @@ from gitdb.test.lib import ( with_rw_directory, with_packs_rw, - ZippedStoreShaWriter, fixture_path, TestBase ) -from gitdb.stream import Sha1Writer + + +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) from gitdb.base import ( IStream, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index cce2b9c09..4bce7dac8 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,7 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + fixture_path, + with_rw_directory +) from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 5e42b639a..1299f7b86 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -2,7 +2,10 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 9235b21d3..97f721719 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -2,7 +2,10 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) from gitdb.db import ( MemoryDB, LooseObjectDB diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f5a4dcb92..1177cf962 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -2,9 +2,12 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + with_packs_rw +) from gitdb.db import PackedDB -from gitdb.test.lib import fixture_path from gitdb.exc import BadObject, AmbiguousObjectName @@ -46,8 +49,8 @@ def test_writing(self, path): random.shuffle(sha_list) for sha in sha_list: - info = pdb.info(sha) - stream = pdb.stream(sha) + pdb.info(sha) + pdb.stream(sha) # END for each sha to query diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index cb77e69bf..d88ec8b43 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -3,14 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" -from gitdb import ( - OStream, - ) -from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter -) - +from gitdb import OStream from gitdb.utils.compat import xrange import sys diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 3563fcfbe..5b5c40e0d 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -4,9 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os -from gitdb.test.lib import * -import shutil -import tempfile +from gitdb.test.lib import TestBase #{ Invvariants diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 63856e218..b18e31ae6 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -9,11 +9,11 @@ from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB +from gitdb.utils.compat import xrange import sys import os from time import time -import random from nose import SkipTest diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index c66e60cba..297426303 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -38,7 +38,6 @@ def test_pack_writing(self): ni = 5000 count = 0 - total_size = 0 st = time() for sha in pdb.sha_iter(): count += 1 diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 010003d4b..6c8f71520 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -4,13 +4,13 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" from lib import TestBigRepoR -from gitdb.db import * -from gitdb.base import * -from gitdb.stream import * +from gitdb.db import LooseObjectDB +from gitdb.stream import IStream + from gitdb.util import ( - pool, - bin_to_hex - ) + pool, + bin_to_hex +) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size @@ -19,19 +19,15 @@ ChannelThreadTask, ) -from cStringIO import StringIO from time import time import os import sys -import stat -import subprocess from lib import ( - TestBigRepoR, make_memory_file, with_rw_directory - ) +) #{ Utilities diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 4cca7dabe..578c29f73 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -9,7 +9,15 @@ DeriveTest, ) -from gitdb import * +from gitdb import ( + OInfo, + OPackInfo, + ODeltaPackInfo, + OStream, + OPackStream, + ODeltaPackStream, + IStream +) from gitdb.util import ( NULL_BIN_SHA ) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index c644b8849..433518c25 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,7 +3,10 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from gitdb.test.lib import * +from gitdb.test.lib import ( + TestBase, + fixture_path +) from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 6b67ad887..3ab2fec07 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -6,7 +6,6 @@ from gitdb.test.lib import ( TestBase, with_rw_directory, - with_packs_rw, fixture_path ) @@ -27,7 +26,6 @@ from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha from gitdb.utils.compat import xrange -from itertools import chain try: from itertools import izip @@ -37,7 +35,6 @@ from nose import SkipTest import os -import sys import tempfile diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 984d6e1d7..671a146da 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -7,17 +7,18 @@ from gitdb.test.lib import ( TestBase, DummyStream, - Sha1Writer, make_bytes, make_object, fixture_path ) -from gitdb import * -from gitdb.util import ( - NULL_HEX_SHA, - hex_to_bin +from gitdb import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + LooseObjectDB, + Sha1Writer ) +from gitdb.util import hex_to_bin import zlib from gitdb.typ import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index ec9a86ce7..e79355aaf 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -5,7 +5,6 @@ """Test for object db""" import tempfile import os -import sys from gitdb.test.lib import TestBase from gitdb.util import ( diff --git a/gitdb/util.py b/gitdb/util.py index 77a0023e6..8843e8c88 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -77,7 +77,10 @@ def unpack_from(fmt, data, offset=0): fsync = os.fsync # Backwards compatibility imports -from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA +from gitdb.const import ( + NULL_BIN_SHA, + NULL_HEX_SHA +) #} END Aliases @@ -124,6 +127,7 @@ def make_sha(source=''.encode("ascii")): try: return hashlib.sha1(source) except NameError: + import sha sha1 = sha.sha(source) return sha1 diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index eec319f3d..a2640fd23 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -6,8 +6,10 @@ from itertools import izip xrange = xrange except ImportError: + # py3 izip = zip xrange = range +# end handle python version try: # Python 2 @@ -15,11 +17,15 @@ memoryview = buffer except NameError: # Python 3 has no `buffer`; only `memoryview` + # However, it's faster to just slice the object directly, maybe it keeps a view internally def buffer(obj, offset, size=None): if size is None: - return memoryview(obj)[offset:] + # return memoryview(obj)[offset:] + return obj[offset:] else: - return memoryview(obj[offset:offset+size]) + # return memoryview(obj)[offset:offset+size] + return obj[offset:offset+size] + # end buffer reimplementation memoryview = memoryview From bf942a913d69eb2079f9e82888aaccf2f6222643 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 13:31:32 +0100 Subject: [PATCH 0316/3719] Fully removed all async dependencies --- .gitmodules | 3 - doc/source/changes.rst | 7 +++ gitdb/__init__.py | 4 +- gitdb/db/base.py | 55 ---------------- gitdb/db/mem.py | 8 +-- gitdb/db/pack.py | 4 -- gitdb/ext/async | 1 - gitdb/test/__init__.py | 12 ---- gitdb/test/db/lib.py | 83 ------------------------ gitdb/test/db/test_git.py | 1 - gitdb/test/db/test_loose.py | 1 - gitdb/test/performance/test_stream.py | 90 +-------------------------- gitdb/test/test_example.py | 23 ------- gitdb/util.py | 10 --- setup.py | 6 +- 15 files changed, 14 insertions(+), 294 deletions(-) delete mode 160000 gitdb/ext/async diff --git a/.gitmodules b/.gitmodules index 978105388..d85b15c9f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "async"] - path = gitdb/ext/async - url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 839bf16a8..f544f76c0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +0.6.0 +***** + +* Added support got python 3.X +* Removed all `async` dependencies and all `*_async` versions of methods with it. + ***** 0.5.4 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 72b5ab085..165993fc9 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -10,7 +10,7 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - for module in ('async', 'smmap'): + for module in ('smmap',): sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) try: @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 5) +version_info = (0, 6, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index c534705b7..eac54ec38 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,7 +4,6 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, join, LazyMixin, hex_to_bin @@ -15,10 +14,6 @@ AmbiguousObjectName ) -from async import ( - ChannelThreadTask -) - from itertools import chain from functools import reduce @@ -41,47 +36,18 @@ def has_object(self, sha): binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") - def has_object_async(self, reader): - """Return a reader yielding information about the membership of objects - as identified by shas - :param reader: Reader yielding 20 byte shas. - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate - whether the given sha exists in the database or not""" - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - def info(self, sha): """ :return: OInfo instance :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def info_async(self, reader): - """Retrieve information of a multitude of objects asynchronously - :param reader: Channel yielding the sha's of the objects of interest - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" - task = ChannelThreadTask(reader, str(self.info_async), self.info) - return pool.add_task(task) - def stream(self, sha): """:return: OStream instance :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def stream_async(self, reader): - """Retrieve the OStream of multiple objects - :param reader: see ``info`` - :param max_threads: see ``ObjectDBW.store`` - :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to - read all OStreams at once. Instead, read them individually using reader.read(x) - where x is small enough.""" - # base implementation just uses the stream method repeatedly - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) - return pool.add_task(task) - def size(self): """:return: amount of objects in this database""" raise NotImplementedError() @@ -129,27 +95,6 @@ def store(self, istream): :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - def store_async(self, reader): - """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as - they are computed. - - :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, - or its error attribute will be set to the exception informing about the error. - - :param reader: async.Reader yielding IStream instances. - The same instances will be used in the output channel as were received - in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how - many items have actually been produced.""" - # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) - return pool.add_task(task) - #} END edit interface diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index a22454685..1aa0d511f 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -32,10 +32,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): """A memory database stores everything to memory, providing fast IO and object retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already - exists in the target storage before introducing actual IO - - **Note:** memory is currently not threadsafe, hence the async methods cannot be used - for storing""" + exists in the target storage before introducing actual IO""" def __init__(self): super(MemoryDB, self).__init__() @@ -62,9 +59,6 @@ def store(self, istream): return istream - def store_async(self, reader): - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - def has_object(self, sha): return sha in self._cache diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 4d0a7f8b3..eaf431a22 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -125,10 +125,6 @@ def store(self, istream): inefficient""" raise UnsupportedOperation() - def store_async(self, reader): - # TODO: add ObjectDBRW before implementing this - raise NotImplementedError() - #} END object db write diff --git a/gitdb/ext/async b/gitdb/ext/async deleted file mode 160000 index b930ee15c..000000000 --- a/gitdb/ext/async +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index ca104c0c5..8a681e428 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -2,15 +2,3 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php - -import gitdb.util - -#{ Initialization -def _init_pool(): - """Assure the pool is actually threaded""" - size = 2 - print("Setting ThreadPool to %i" % size) - gitdb.util.pool.set_size(size) - - -#} END initialization diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 67958c366..962d4bc2a 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -10,8 +10,6 @@ TestBase ) - - from gitdb.stream import ( Sha1Writer, ZippedStoreShaWriter @@ -28,8 +26,6 @@ from gitdb.utils.encoding import force_bytes from gitdb.utils.compat import xrange -from async import IteratorReader - from io import BytesIO from struct import pack @@ -132,82 +128,3 @@ def _assert_object_writing(self, db): # END for each data set # END for each dry_run mode - def _assert_object_writing_async(self, db): - """Test generic object writing using asynchronous access""" - ni = 5000 - def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - data = bytes(data_src + offset) - yield IStream(str_blob_type, len(data), BytesIO(data)) - # END for each item - # END generator utility - - # for now, we are very trusty here as we expect it to work if it worked - # in the single-stream case - - # write objects - reader = IteratorReader(istream_generator()) - istream_reader = db.store_async(reader) - istreams = istream_reader.read() # read all - assert istream_reader.task().error() is None - assert len(istreams) == ni - - for stream in istreams: - assert stream.error is None - assert len(stream.binsha) == 20 - assert isinstance(stream, IStream) - # END assert each stream - - # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.binsha for istream in istreams ) - hasobject_reader = db.has_object_async(reader) - count = 0 - for sha, has_object in hasobject_reader: - assert has_object - count += 1 - # END for each sha - assert count == ni - - # read the objects we have just written - reader = IteratorReader( istream.binsha for istream in istreams ) - ostream_reader = db.stream_async(reader) - - # read items individually to prevent hitting possible sys-limits - count = 0 - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert ostream_reader.task().error() is None - assert count == ni - - # get info about our items - reader = IteratorReader( istream.binsha for istream in istreams ) - info_reader = db.info_async(reader) - - count = 0 - for oinfo in info_reader: - assert isinstance(oinfo, OInfo) - count += 1 - # END for each oinfo instance - assert count == ni - - - # combined read-write using a converter - # add 2500 items, and obtain their output streams - nni = 2500 - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - - istream_reader = db.store_async(reader) - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = db.stream_async(istream_reader) - - count = 0 - # read it individually, otherwise we might run into the ulimit - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert count == nni diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 4bce7dac8..56899e579 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -48,4 +48,3 @@ def test_writing(self, path): # its possible to write objects self._assert_object_writing(gdb) - self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 1299f7b86..1d6af9c99 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -18,7 +18,6 @@ def test_basics(self, path): # write data self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) # verify sha iteration and size shas = list(ldb.sha_iter()) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 6c8f71520..929c7e537 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -7,18 +7,9 @@ from gitdb.db import LooseObjectDB from gitdb.stream import IStream -from gitdb.util import ( - pool, - bin_to_hex -) -from gitdb.typ import str_blob_type +from gitdb.util import bin_to_hex from gitdb.fun import chunk_size -from async import ( - IteratorReader, - ChannelThreadTask, - ) - from time import time import os import sys @@ -43,15 +34,6 @@ def read_chunked_stream(stream): return stream -class TestStreamReader(ChannelThreadTask): - """Expects input streams and reads them in chunks. It will read one at a time, - requireing a queue chunk of size 1""" - def __init__(self, *args): - super(TestStreamReader, self).__init__(*args) - self.fun = read_chunked_stream - self.max_chunksize = 1 - - #} END utilities class TestObjDBPerformance(TestBigRepoR): @@ -119,73 +101,3 @@ def test_large_data_streaming(self, path): # del db file so we keep something to do os.remove(db_file) # END for each randomization factor - - - # multi-threaded mode - # want two, should be supported by most of todays cpus - pool.set_size(2) - total_kib = 0 - nsios = len(string_ios) - for stream in string_ios: - stream.seek(0) - total_kib += len(stream.getvalue()) / 1000 - # END rewind - - def istream_iter(): - for stream in string_ios: - stream.seek(0) - yield IStream(str_blob_type, len(stream.getvalue()), stream) - # END for each stream - # END util - - # write multiple objects at once, involving concurrent compression - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - st = time() - istreams = istream_reader.read(nsios) - assert len(istreams) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # decompress multiple at once, by reading them - # chunk size is not important as the stream will not really be decompressed - - # until its read - istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.task().max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # store the files, and read them back. For the reading, we use a task - # as well which is chunked into one item per task. Reading all will - # very quickly result in two threads handling two bytestreams of - # chained compression/decompression streams - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - istream_to_sha = lambda items: [ i.binsha for i in items ] - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 433518c25..aa43a093f 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -9,12 +9,9 @@ ) from gitdb import IStream from gitdb.db import LooseObjectDB -from gitdb.util import pool from io import BytesIO -from async import IteratorReader - class TestExamples(TestBase): def test_base(self): @@ -45,23 +42,3 @@ def test_base(self): # now the sha is set assert len(istream.binsha) == 20 assert ldb.has_object(istream.binsha) - - - # async operation - # Create a reader from an iterator - reader = IteratorReader(ldb.sha_iter()) - - # get reader for object streams - info_reader = ldb.stream_async(reader) - - # read one - info = info_reader.read(1)[0] - - # read all the rest until depletion - ostreams = info_reader.read() - - # set the pool to use two threads - pool.set_size(2) - - # synchronize the mode of operation - pool.set_size(0) diff --git a/gitdb/util.py b/gitdb/util.py index 8843e8c88..93ba7f0ec 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -10,7 +10,6 @@ from io import StringIO -from async import ThreadPool from smmap import ( StaticWindowMapManager, SlidingWindowMapManager, @@ -43,15 +42,6 @@ def unpack_from(fmt, data, offset=0): # END own unpack_from implementation -#{ Globals - -# A pool distributing tasks, initially with zero threads, hence everything -# will be handled in the main thread -pool = ThreadPool(0) - -#} END globals - - #{ Aliases hex_to_bin = binascii.a2b_hex diff --git a/setup.py b/setup.py index 639370471..63ec5ddb3 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 5) +version_info = (0, 6, 0) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, @@ -88,8 +88,8 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.2)', 'smmap (>=0.8.3)'), - install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), + requires=('smmap (>=0.8.3)'), + install_requires=('smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 641b64c9f48139cf06774805d32892012fb9b82d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 18:31:17 +0100 Subject: [PATCH 0317/3719] Now tests work consistently in py2 and 3 It's a nice way of saying that there is still one failing, consistently. --- gitdb/const.py | 5 +- gitdb/db/base.py | 7 +- gitdb/db/loose.py | 2 +- gitdb/fun.py | 229 +++++++++++++++++++++++++------------ gitdb/pack.py | 21 ++-- gitdb/stream.py | 8 +- gitdb/test/db/test_pack.py | 2 +- gitdb/typ.py | 10 +- gitdb/utils/encoding.py | 4 +- 9 files changed, 178 insertions(+), 110 deletions(-) diff --git a/gitdb/const.py b/gitdb/const.py index 147f79cb9..6391d796f 100644 --- a/gitdb/const.py +++ b/gitdb/const.py @@ -1,5 +1,4 @@ -from gitdb.utils.encoding import force_bytes - -NULL_BYTE = force_bytes("\0") +BYTE_SPACE = b' ' +NULL_BYTE = b'\0' NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index eac54ec38..a670eea63 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -9,6 +9,7 @@ hex_to_bin ) +from gitdb.utils.encoding import force_text from gitdb.exc import ( BadObject, AmbiguousObjectName @@ -122,8 +123,6 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" - from gitdb.utils.encoding import force_text - return join(self._root_path, force_text(rela_path)) #} END interface @@ -234,12 +233,12 @@ def update_cache(self, force=False): def partial_to_complete_sha_hex(self, partial_hexsha): """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str) :param partial_hexsha: hexsha with less than 40 byte :raise AmbiguousObjectName: """ databases = list() _databases_recursive(self, databases) - + partial_hexsha = force_text(partial_hexsha) len_partial_hexsha = len(partial_hexsha) if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 3abdaa96f..374302611 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -109,7 +109,7 @@ def readable_db_object_path(self, hexsha): def partial_to_complete_sha_hex(self, partial_hexsha): """:return: 20 byte binary sha1 string which matches the given name uniquely - :param name: hexadecimal partial name + :param name: hexadecimal partial name (bytes or ascii string) :raise AmbiguousObjectName: :raise BadObject: """ candidate = None diff --git a/gitdb/fun.py b/gitdb/fun.py index 80f01e6b2..064680adb 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -14,7 +14,9 @@ from itertools import islice from functools import reduce -from gitdb.utils.compat import izip, buffer, xrange +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_text +from gitdb.utils.compat import izip, buffer, xrange, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -30,12 +32,12 @@ delta_types = (OFS_DELTA, REF_DELTA) type_id_to_type_map = { - 0 : "", # EXT 1 + 0 : b'', # EXT 1 1 : str_commit_type, 2 : str_tree_type, 3 : str_blob_type, 4 : str_tag_type, - 5 : "", # EXT 2 + 5 : b'', # EXT 2 OFS_DELTA : "OFS_DELTA", # OFFSET DELTA REF_DELTA : "REF_DELTA" # REFERENCE DELTA } @@ -394,11 +396,9 @@ def loose_object_header_info(m): :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" - from gitdb.const import NULL_BYTE - decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE) return type_name, int(size) @@ -413,12 +413,21 @@ def pack_object_header_info(data): type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size - while c & 0x80: - c = byte_ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop + if PY3: + while c & 0x80: + c = data[i] + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + else: + while c & 0x80: + c = ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + # end performance at expense of maintenance ... return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): @@ -429,16 +438,29 @@ def create_pack_object_header(obj_type, obj_size): :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte - hdr = str() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - #END until size is consumed - hdr += chr(c) + if PY3: + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr.append(c) + else: + hdr = bytes() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + # end handle interpreter return hdr def msb_size(data, offset=0): @@ -449,24 +471,36 @@ def msb_size(data, offset=0): i = 0 l = len(data) hit_msb = False - while i < l: - c = byte_ord(data[i+offset]) - size |= (c & 0x7f) << i*7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range + if PY3: + while i < l: + c = data[i+offset] + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + else: + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") return i+offset, size def loose_object_header(type, size): """ - :return: string representing the loose object header, which is immediately + :return: bytes representing the loose object header, which is immediately followed by the content stream of size 'size'""" - return "%s %i\0" % (type, size) + return ('%s %i\0' % (force_text(type), size)).encode('ascii') def write_object(type, size, read, write, chunk_size=chunk_size): """ @@ -611,48 +645,93 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i+c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data + if PY3: + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + else: + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + # end save byte_ord call and prevent performance regression in py2 # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" diff --git a/gitdb/pack.py b/gitdb/pack.py index 87818b6e7..375cc59a9 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -86,8 +86,6 @@ def pack_object_at(cursor, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - from gitdb.utils.encoding import force_bytes - data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -96,11 +94,11 @@ def pack_object_at(cursor, offset, as_stream): # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 delta_offset = c & 0x7f while c & 0x80: - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 delta_offset += 1 delta_offset = (delta_offset << 7) + (c & 0x7f) @@ -118,12 +116,10 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - buff = buffer(data, total_rela_offset) - stream = DecompressMemMapReader(buff, False, uncomp_size) + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - delta_info = force_bytes(delta_info) return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: @@ -204,7 +200,7 @@ def write(self, pack_sha, write): # fanout tmplist = list((0,)*256) # fanout or list with 64 bit offsets for t in self._objs: - tmplist[ord(t[0][0])] += 1 + tmplist[byte_ord(t[0][0])] += 1 #END prepare fanout for i in xrange(255): v = tmplist[i] @@ -215,7 +211,7 @@ def write(self, pack_sha, write): # sha1 ordered # save calls, that is push them into c - sha_write(''.join(t[0] for t in self._objs)) + sha_write(b''.join(t[0] for t in self._objs)) # crc32 for t in self._objs: @@ -258,7 +254,7 @@ class PackIndexFile(LazyMixin): # used in v2 indices _sha_list_offset = 8 + 1024 - index_v2_signature = '\377tOc' + index_v2_signature = b'\xfftOc' index_version_default = 2 def __init__(self, indexpath): @@ -446,13 +442,14 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this index file - :param partial_bin_sha: an at least two bytes of a partial binary sha + :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes :param canonical_length: lenght of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") + assert isinstance(partial_bin_sha, bytes), "partial_bin_sha must be bytes" first_byte = byte_ord(partial_bin_sha[0]) get_sha = self.sha @@ -680,7 +677,7 @@ def _set_cache_(self, attr): else: iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) - iter_offsets_plus_one.next() + next(iter_offsets_plus_one) consecutive = izip(iter_offsets, iter_offsets_plus_one) offset_map = dict(consecutive) diff --git a/gitdb/stream.py b/gitdb/stream.py index 5daf01cb6..43aa8e368 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -25,7 +25,7 @@ close, ) -from gitdb.const import NULL_BYTE +from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.compat import buffer from gitdb.utils.encoding import force_bytes @@ -102,7 +102,7 @@ def _parse_header_info(self): self._s = maxb hdr = self.read(maxb) hdrend = hdr.find(NULL_BYTE) - typ, size = hdr[:hdrend].split(" ".encode("ascii")) + typ, size = hdr[:hdrend].split(BYTE_SPACE) size = int(size) self._s = size @@ -378,8 +378,6 @@ def _set_cache_too_slow_without_c(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" - from gitdb.utils.compat import buffer - # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the # delta is not peaked into, causing less overhead. @@ -421,7 +419,7 @@ def _set_cache_brute_(self, attr): # For the actual copying, we use a seek and write pattern of buffer # slices. final_target_size = None - for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): + for (dbuf, offset, src_size, target_size), dstream in zip(reversed(buffer_info_list), reversed(self._dstreams)): # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 1177cf962..963a71af7 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -73,4 +73,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/typ.py b/gitdb/typ.py index edd1f2731..bc7ba5828 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,9 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -from gitdb.utils.encoding import force_bytes - -str_blob_type = force_bytes("blob") -str_commit_type = force_bytes("commit") -str_tree_type = force_bytes("tree") -str_tag_type = force_bytes("tag") +str_blob_type = b'blob' +str_commit_type = b'commit' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 12164e756..617b51c83 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -11,9 +11,6 @@ def force_bytes(data, encoding="utf-8"): if isinstance(data, bytes): return data - if isinstance(data, compat.memoryview): - return bytes(data) - if isinstance(data, string_types): return data.encode(encoding) @@ -27,6 +24,7 @@ def force_text(data, encoding="utf-8"): return data.decode(encoding) if not isinstance(data, bytes): + assert False, "Shouldn't be here" data = force_bytes(data, encoding) if compat.PY3: From 8ae4e9579a263684c6b760aec2869be480ff22ba Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 18:46:49 +0100 Subject: [PATCH 0318/3719] reduced usage of force_bytes as clients are expected to pass bytes. It was useful for debugging though, maybe an explicit type assertions would help others ? As 'others' will be gitpython, I suppose I can handle it myself --- gitdb/stream.py | 6 +----- gitdb/test/db/lib.py | 7 +++---- gitdb/test/db/test_ref.py | 8 +++++--- gitdb/utils/encoding.py | 6 +----- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 43aa8e368..e32fcf380 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -543,9 +543,9 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written + :param data: byte object :return: length of incoming data""" - data = force_bytes(data) self.sha1.update(data) return len(data) @@ -590,8 +590,6 @@ def __getattr__(self, attr): return getattr(self.buf, attr) def write(self, data): - data = force_bytes(data) - alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) @@ -634,8 +632,6 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written :return: lenght of incoming data""" - data = force_bytes(data) - self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 962d4bc2a..af6d9e0fd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,6 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type -from gitdb.utils.encoding import force_bytes from gitdb.utils.compat import xrange from io import BytesIO @@ -37,7 +36,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world" + two_lines = b'1234\nhello world' all_data = (two_lines, ) def _assert_object_writing_simple(self, db): @@ -83,7 +82,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii"))) + istream = IStream(str_blob_type, len(data), BytesIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -99,7 +98,7 @@ def _assert_object_writing(self, db): assert info.size == len(data) ostream = db.stream(sha) - assert ostream.read() == force_bytes(data) + assert ostream.read() == data assert ostream.type == str_blob_type assert ostream.size == len(data) else: diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index a1387ee26..db930827b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -2,7 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + fixture_path +) from gitdb.db import ReferenceDB from gitdb.util import ( @@ -24,8 +28,6 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - NULL_BIN_SHA = '\0'.encode("ascii") * 20 - alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) assert len(rdb.databases()) == 0 diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 617b51c83..2d03ad30c 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -7,7 +7,7 @@ string_types = (basestring, ) text_type = unicode -def force_bytes(data, encoding="utf-8"): +def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -23,10 +23,6 @@ def force_text(data, encoding="utf-8"): if isinstance(data, string_types): return data.decode(encoding) - if not isinstance(data, bytes): - assert False, "Shouldn't be here" - data = force_bytes(data, encoding) - if compat.PY3: return text_type(data, encoding) else: From 7fd369c8549a975efd74c312aa91194b4569b99e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 19:49:40 +0100 Subject: [PATCH 0319/3719] setup.py works now, and binary python module can now be loaded as well. --- gitdb/_delta_apply.c | 2 -- gitdb/fun.py | 1 - setup.py | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index f03e7ea6d..8b0f8e064 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -506,7 +506,6 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded // and added to the spot marked by sdc -inline uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) { uint num_bytes = 0; @@ -559,7 +558,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) // destination memory. The individual chunks written will be a byte copy of the source // data chunk stream // Return: number of chunks in the slice -inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) { assert(DIV_lbound(src) <= tofs); diff --git a/gitdb/fun.py b/gitdb/fun.py index 064680adb..b7662b495 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -758,7 +758,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - # NOQA from _perf import connect_deltas except ImportError: pass diff --git a/setup.py b/setup.py index 63ec5ddb3..e01c6b475 100755 --- a/setup.py +++ b/setup.py @@ -83,12 +83,12 @@ def get_data_files(self): author = __author__, author_email = __contact__, url = __homepage__, - packages = ('gitdb', 'gitdb.db'), + packages = ('gitdb', 'gitdb.db', 'gitdb.utils'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('smmap (>=0.8.3)'), + requires=('smmap (>=0.8.3)', ), install_requires=('smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From f26c869025bc31d3920726a63b15fdb420b1d215 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 09:43:19 +0100 Subject: [PATCH 0320/3719] Performance tests are now part of the test-suite. By default, a small repository will be tested, which doesn't take that long actually (~20s) Additionally, that way we enforce correctness tests, which didn't run by default previously. As we are handling data here, we must be sure that it's handled correctly, thus the tests should run. --- gitdb/test/lib.py | 4 +-- gitdb/test/performance/lib.py | 27 ++++++++----------- gitdb/test/performance/test_pack.py | 21 +++++++-------- gitdb/test/performance/test_pack_streaming.py | 17 ++++++------ gitdb/test/performance/test_stream.py | 20 +++++++------- 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index d88ec8b43..ba653c91f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -10,7 +10,7 @@ import random from array import array -from io import StringIO +from io import BytesIO import glob import unittest @@ -120,7 +120,7 @@ def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) + return len(d), BytesIO(d) #} END routines diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 5b5c40e0d..ec45cf3a7 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -4,6 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os +import logging from gitdb.test.lib import TestBase @@ -12,17 +13,6 @@ #} END invariants -#{ Utilities -def resolve_or_fail(env_var): - """:return: resolved environment variable or raise EnvironmentError""" - try: - return os.environ[env_var] - except KeyError: - raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) - # END exception handling - -#} END utilities - #{ Base Classes @@ -39,14 +29,19 @@ class TestBigRepoR(TestBase): head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' #} END invariants - @classmethod - def setUpAll(cls): + def setUp(self): try: - super(TestBigRepoR, cls).setUpAll() + super(TestBigRepoR, self).setUp() except AttributeError: pass - cls.gitrepopath = resolve_or_fail(k_env_git_repo) - assert cls.gitrepopath.endswith('.git') + + self.gitrepopath = os.environ.get(k_env_git_repo) + if not self.gitrepopath: + logging.info("You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository") + ospd = os.path.dirname + self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') + # end assure gitrepo is set + assert self.gitrepopath.endswith('.git') #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b18e31ae6..b52e46fec 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" -from lib import ( +from __future__ import print_function + +from gitdb.test.performance.lib import ( TestBigRepoR - ) +) from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB @@ -15,8 +17,6 @@ import os from time import time -from nose import SkipTest - class TestPackedDBPerformance(TestBigRepoR): def test_pack_random_access(self): @@ -27,7 +27,7 @@ def test_pack_random_access(self): sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) - print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info @@ -41,7 +41,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) # END for each random mode # query info and streams only @@ -51,7 +51,7 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) # END for each function # retrieve stream and read all @@ -65,13 +65,12 @@ def test_pack_random_access(self): total_size += stream.size elapsed = time() - st total_kib = total_size / 1000 - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) def test_correctness(self): - raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time - print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" + print("Endurance run: verify streaming of objects (crc and sha)", file=sys.stderr) for crc in range(2): count = 0 st = time() @@ -88,6 +87,6 @@ def test_correctness(self): # END for each index # END for each entity elapsed = time() - st - print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed), file=sys.stderr) # END for each verify mode diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 297426303..b1001f85b 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" -from lib import ( +from __future__ import print_function + +from gitdb.test.performance.lib import ( TestBigRepoR - ) +) from gitdb.db.pack import PackedDB from gitdb.stream import NullStream @@ -14,7 +16,6 @@ import os import sys from time import time -from nose import SkipTest class CountedNullStream(NullStream): __slots__ = '_bw' @@ -36,7 +37,7 @@ def test_pack_writing(self): ostream = CountedNullStream() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - ni = 5000 + ni = 1000 count = 0 st = time() for sha in pdb.sha_iter(): @@ -46,17 +47,17 @@ def test_pack_writing(self): break #END gather objects for pack-writing elapsed = time() - st - print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed), file=sys.stderr) st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 - print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) def test_stream_reading(self): - raise SkipTest() + # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # streaming only, meant for --with-profile runs @@ -74,5 +75,5 @@ def test_stream_reading(self): count += 1 elapsed = time() - st total_kib = total_size / 1000 - print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 929c7e537..9d695a047 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" -from lib import TestBigRepoR +from __future__ import print_function + +from gitdb.test.performance.lib import TestBigRepoR from gitdb.db import LooseObjectDB -from gitdb.stream import IStream +from gitdb import IStream from gitdb.util import bin_to_hex from gitdb.fun import chunk_size @@ -15,7 +17,7 @@ import sys -from lib import ( +from gitdb.test.lib import ( make_memory_file, with_rw_directory ) @@ -49,11 +51,11 @@ def test_large_data_streaming(self, path): # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' - print >> sys.stderr, "Creating %s data ..." % desc + print("Creating %s data ..." % desc, file=sys.stderr) st = time() size, stream = make_memory_file(self.large_data_size_bytes, randomize) elapsed = time() - st - print >> sys.stderr, "Done (in %f s)" % elapsed + print("Done (in %f s)" % elapsed, file=sys.stderr) string_ios.append(stream) # writing - due to the compression it will seem faster than it is @@ -66,7 +68,7 @@ def test_large_data_streaming(self, path): size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) # reading all at once st = time() @@ -76,7 +78,7 @@ def test_large_data_streaming(self, path): stream.seek(0) assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) # reading in chunks of 1 MiB @@ -93,10 +95,10 @@ def test_large_data_streaming(self, path): elapsed_readchunks = time() - st stream.seek(0) - assert ''.join(chunks) == stream.getvalue() + assert b''.join(chunks) == stream.getvalue() cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) # del db file so we keep something to do os.remove(db_file) From 232cf20efaef4218a71c348b0f59c5e59c59cbdb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 10:43:28 +0100 Subject: [PATCH 0321/3719] Fixed incorrect computation of compressed bytes read in zlib decompression stream. --- gitdb/stream.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index e32fcf380..0332df6e4 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -275,7 +275,10 @@ def read(self, size=-1): # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data # if we hit the end of the stream - self._cbr += len(indata) - len(self._zip.unconsumed_tail) + # NOTE: For some reason, the code worked for a long time with substracting unconsumed_tail + # Now, however, it really asks for unused_data, and I wonder whether unconsumed_tail still has to + # be substracted. On the plus side, the tests work, so it seems to be ok for py 2.7 and 3.4 + self._cbr += len(indata) - len(self._zip.unconsumed_tail) - len(self._zip.unused_data) self._br += len(dcompdat) if dat: From 6f71b8a90250ed03f2e7de2e61d22e84c0fbb2ff Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:04:23 +0100 Subject: [PATCH 0322/3719] Fixed .travis file to allow tests to work correctly. Previously, submodules were not initalized, which could have had an effect ... . Even though it shouldn't, but lets just try it. --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf1d13666..db7c2bc66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,11 @@ python: - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) -script: nosetests +install: + - git submodule update --init --recursive + - pip install coveralls +script: + - nosetests +after_success: + - coveralls + From b9d189d35073cc80ddbfa61269c65785264880f3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:29:18 +0100 Subject: [PATCH 0323/3719] Added requirements.txt for pip, and optimized test-suite performance on travis. With a bit of luck, this one will just work now. --- .travis.yml | 1 - README.rst | 33 +++++++++++-------- doc/source/intro.rst | 8 ++--- gitdb/test/db/test_git.py | 4 ++- gitdb/test/lib.py | 16 +++++++++ gitdb/test/performance/__init__.py | 1 + gitdb/test/performance/test_pack.py | 5 ++- gitdb/test/performance/test_pack_streaming.py | 3 ++ gitdb/test/performance/test_stream.py | 6 ++-- requirements.txt | 2 ++ setup.py | 2 +- 11 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 gitdb/test/performance/__init__.py create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index db7c2bc66..10b5e161b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) install: - - git submodule update --init --recursive - pip install coveralls script: - nosetests diff --git a/README.rst b/README.rst index f97d5c31e..194e24658 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,7 @@ GitDB ===== -GitDB allows you to access bare git repositories for reading and writing. It -aims at allowing full access to loose objects as well as packs with performance -and scalability in mind. It operates exclusively on streams, allowing to operate -on large objects with a small memory footprint. +GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. Installation ============ @@ -23,13 +20,13 @@ From `PyPI `_ REQUIREMENTS ============ -* Python Nose - for running the tests +* Python Nose - for running the tests SOURCE ====== The source is available in a git repository at gitorious and github: -git://github.com/gitpython-developers/gitdb.git +https://github.com/gitpython-developers/gitdb Once the clone is complete, please be sure to initialize the submodules using @@ -40,17 +37,25 @@ Run the tests with nosetests -MAILING LIST -============ -http://groups.google.com/group/git-python - -ISSUE TRACKER -============= +DEVELOPMENT +=========== .. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - -https://github.com/gitpython-developers/gitdb/issues + +.. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png + :target: https://coveralls.io/r/gitpython-developers/gitdb + +The library is considered mature, and not under active development. It's primary (known) use is in git-python. + +INFRASTRUCTURE +============== + +* Mailing List + * http://groups.google.com/group/git-python + +* Issue Tracker + * https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8fc0ec098..434138616 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -11,9 +11,9 @@ Interfaces are used to describe the API, making it easy to provide alternate imp ================ Installing GitDB ================ -Its easiest to install gitdb using the *easy_install* program, which is part of the `setuptools`_:: +Its easiest to install gitdb using the *pip* program:: - $ easy_install gitdb + $ pip install gitdb As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. @@ -31,10 +31,8 @@ Source Repository ================= The latest source can be cloned using git from github: - * git://github.com/gitpython-developers/gitdb.git + * https://github.com/gitpython-developers/gitdb License Information =================== *GitDB* is licensed under the New BSD License. - -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 56899e579..e141c2ba0 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -24,9 +24,11 @@ def test_reading(self): gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) - assert gdb.size() > 200 + ni = 50 + assert gdb.size() >= ni sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() + sha_list = sha_list[:ni] # speed up tests ... # This is actually a test for compound functionality, but it doesn't diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ba653c91f..d09b1cb8e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -18,6 +18,7 @@ import shutil import os import gc +from functools import wraps #{ Bases @@ -30,6 +31,21 @@ class TestBase(unittest.TestCase): #{ Decorators +def skip_on_travis_ci(func): + """All tests decorated with this one will raise SkipTest when run on travis ci. + Use it to workaround difficult to solve issues + NOTE: copied from bcore (https://github.com/Byron/bcore)""" + @wraps(func) + def wrapper(self, *args, **kwargs): + if 'TRAVIS' in os.environ: + import nose + raise nose.SkipTest("Cannot run on travis-ci") + # end check for travis ci + return func(self, *args, **kwargs) + # end wrapper + return wrapper + + def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" diff --git a/gitdb/test/performance/__init__.py b/gitdb/test/performance/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/gitdb/test/performance/__init__.py @@ -0,0 +1 @@ + diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b52e46fec..db3b48de5 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -12,13 +12,15 @@ from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB from gitdb.utils.compat import xrange +from gitdb.test.lib import skip_on_travis_ci import sys import os from time import time class TestPackedDBPerformance(TestBigRepoR): - + + @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -67,6 +69,7 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index b1001f85b..fe160ea54 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -12,6 +12,7 @@ from gitdb.db.pack import PackedDB from gitdb.stream import NullStream from gitdb.pack import PackEntity +from gitdb.test.lib import skip_on_travis_ci import os import sys @@ -31,6 +32,7 @@ def write(self, d): class TestPackStreamingPerformance(TestBigRepoR): + @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well @@ -56,6 +58,7 @@ def test_pack_writing(self): print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) + @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 9d695a047..84c9dea3f 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -19,7 +19,8 @@ from gitdb.test.lib import ( make_memory_file, - with_rw_directory + with_rw_directory, + skip_on_travis_ci ) @@ -42,7 +43,8 @@ class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*50 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - + + @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..8a4cd3979 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +gitdb +smmap>=0.8.3 \ No newline at end of file diff --git a/setup.py b/setup.py index e01c6b475..dc142c518 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def get_data_files(self): license = "BSD License", zip_safe=False, requires=('smmap (>=0.8.3)', ), - install_requires=('smmap >= 0.8.0'), + install_requires=('smmap >= 0.8.3'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 0bb576427f899631fbbbb822c6d3058174f96847 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:37:12 +0100 Subject: [PATCH 0324/3719] Allow our clone to be deeper to help tests to work --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 10b5e161b..8a9e0afa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ python: - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) +git: + # a higher depth is needed for one of the tests - lets fet + depth: 1000 install: - pip install coveralls script: From e7fdd949d0cb2c42c9217e3c7009eb28c6b53446 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 12:07:13 +0100 Subject: [PATCH 0325/3719] Now I am skipping a problematic test on travis CI. Maybe I can find a py 2.6 interpreter somewhere to reproduce it. --- .travis.yml | 2 +- gitdb/test/test_stream.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a9e0afa6..761edc19b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ git: install: - pip install coveralls script: - - nosetests + - nosetests -v after_success: - coveralls diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 671a146da..f8d9f5dcd 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -27,7 +27,8 @@ import tempfile import os - +import sys +from nose import SkipTest class TestStream(TestBase): """Test stream classes""" @@ -70,10 +71,16 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle special type def test_decompress_reader(self): + cache = dict() for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: - cdata = make_bytes(ds, randomize=False) + if ds in cache: + cdata = cache[ds] + else: + cdata = make_bytes(ds, randomize=False) + cache[ds] = cdata + # end handle caching (maybe helps on py2.6 ?) # zdata = zipped actual data # cdata = original content data @@ -121,6 +128,9 @@ def test_sha_writer(self): assert writer.sha() != prev_sha def test_compressed_writer(self): + if sys.version_info[:2] < (2,7) and os.environ.get('TRAVIS'): + raise SkipTest("For some reason, this test STALLS on travis ci on py2.6, but works on my centos py2.6 interpreter") + # end special case ... for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) From 0dcec5a27b341ce58e5ab169f91aa25b2cafec0c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 12:39:09 +0100 Subject: [PATCH 0326/3719] It seems zlib works differently in py26, and thus requires special handling. This also explains why the tests suddenly stopped working - after all, the interpreter changed ... . --- gitdb/stream.py | 15 ++++++++++----- gitdb/test/test_stream.py | 13 +------------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 0332df6e4..edd6dd2b1 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -7,6 +7,7 @@ import mmap import os +import sys import zlib from gitdb.fun import ( @@ -30,6 +31,7 @@ from gitdb.utils.encoding import force_bytes has_perf_mod = False +PY26 = sys.version_info[:2] < (2, 7) try: from _perf import apply_delta as c_apply_delta has_perf_mod = True @@ -275,10 +277,14 @@ def read(self, size=-1): # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data # if we hit the end of the stream - # NOTE: For some reason, the code worked for a long time with substracting unconsumed_tail - # Now, however, it really asks for unused_data, and I wonder whether unconsumed_tail still has to - # be substracted. On the plus side, the tests work, so it seems to be ok for py 2.7 and 3.4 - self._cbr += len(indata) - len(self._zip.unconsumed_tail) - len(self._zip.unused_data) + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + if PY26: + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # end handle very special case ... + self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) if dat: @@ -505,7 +511,6 @@ def new(cls, stream_list): if stream_list[-1].type_id in delta_types: raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream - return cls(stream_list) #} END interface diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f8d9f5dcd..50db44b1d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -27,8 +27,6 @@ import tempfile import os -import sys -from nose import SkipTest class TestStream(TestBase): """Test stream classes""" @@ -71,16 +69,10 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle special type def test_decompress_reader(self): - cache = dict() for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: - if ds in cache: - cdata = cache[ds] - else: - cdata = make_bytes(ds, randomize=False) - cache[ds] = cdata - # end handle caching (maybe helps on py2.6 ?) + cdata = make_bytes(ds, randomize=False) # zdata = zipped actual data # cdata = original content data @@ -128,9 +120,6 @@ def test_sha_writer(self): assert writer.sha() != prev_sha def test_compressed_writer(self): - if sys.version_info[:2] < (2,7) and os.environ.get('TRAVIS'): - raise SkipTest("For some reason, this test STALLS on travis ci on py2.6, but works on my centos py2.6 interpreter") - # end special case ... for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) From 59589e0fddfc9f3ea318dd7b3c0ef5b4e1bb665c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 14:57:01 +0100 Subject: [PATCH 0327/3719] Added pypi badges [ skip ci ] --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 327d66372..8c9ae4251 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap +[![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) +[![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) + Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: ```bash From d3e5d9cda8eae5b0f19ac25efada6d0b3b9e04e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 15:16:22 +0100 Subject: [PATCH 0328/3719] Updated README to better represent current state --- README.md | 61 ++++++++++++++++++-------------------------- git/test/test_git.py | 13 +++++----- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index d2a858bf9..a3800f921 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ The object database implementation is optimized for handling large quantities of ### INSTALL +[![Latest Version](https://pypip.in/version/GitPython/badge.svg)](https://pypi.python.org/pypi/GitPython/) +[![Supported Python Versions](https://pypip.in/py_versions/GitPython/badge.svg)](https://pypi.python.org/pypi/GitPython/) + If you have downloaded the source code: python setup.py install @@ -28,6 +31,23 @@ A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython +### SOURCE + +GitPython's git repo is available on GitHub, which can be browsed at [github](https://github.com/gitpython-developers/GitPython) and cloned like that: + + git clone git://github.com/gitpython-developers/GitPython.git git-python + + +### INFRASTRUCTURE + +* [User Documentation](http://packages.python.org/GitPython/) +* [Mailing List](http://groups.google.com/group/git-python) +* [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) + +### LICENSE + +New BSD License. See the LICENSE file. + ### DEVELOPMENT STATUS [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg?branch=0.3)](https://travis-ci.org/gitpython-developers/GitPython) @@ -35,20 +55,21 @@ A distribution package can be obtained for manual installation at: The project was idle for 2 years, the last release (v0.3.2 RC1) was made on July 2011. Reason for this might have been the project's dependency on me as sole active maintainer, which is an issue in itself. -Now I am back and fully dedicated to pushing [OSS](https://github.com/Byron/bcore) forward in the realm of [digital content creation](http://gooseberry.blender.org/), and git-python will see some of my time as well. Therefore it will be moving forward, slowly but steadily. +Now that there seems to be a massive user base, this should be motivation enough to let git-python return to a proper state, which means + +* no open pull requests +* no open issues describing bugs -In short, I want to make a new release of 0.3 with all contributions and fixes included, foster community building to facilitate contributions. Everything else is future. +In short, I want to make a new release of 0.3 with all contributions and fixes included, foster community building to facilitate contributions. #### PRESENT GOALS The goals I have set for myself, in order, are as follows, all on branch 0.3. * bring the test suite back online to work with the most commonly used git version -* setup a travis test-matrix to test against a lower and upper git version as well * merge all open pull requests, may there be a test-case or not, back. If something breaks, fix it if possible or let the contributor know * conform git-python's structure and toolchain to the one used in my [other OSS projects](https://github.com/Byron/bcore) * evaluate all open issues and close them if possible -* create a new release of the 0.3 branch * evaluate python 3.3 compatibility and establish it if possible While that is happening, I will try hard to foster community around the project. This means being more responsive on the mailing list and in issues, as well as setting up clear guide lines about the [contribution](http://rfc.zeromq.org/spec:22) and maintenance workflow. @@ -63,35 +84,3 @@ There has been a lot of work in the master branch, which is the direction I want * make it work similarly to 0.3, but with the option to swap for at least one additional backend * make a 1.0 release * add backends as required - -### SOURCE - - -GitPython's git repo is available on GitHub, which can be browsed at: - -https://github.com/gitpython-developers/GitPython - -and cloned using: - -git clone git://github.com/gitpython-developers/GitPython.git git-python - - -### DOCUMENTATION - -The html-compiled documentation can be found at the following URL: - -http://packages.python.org/GitPython/ - -### MAILING LIST - -http://groups.google.com/group/git-python - -### ISSUE TRACKER - -Issues are tracked on github: - -https://github.com/gitpython-developers/GitPython/issues - -### LICENSE - -New BSD License. See the LICENSE file. diff --git a/git/test/test_git.py b/git/test/test_git.py index 5d4756baf..cdea1d3e1 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -4,10 +4,9 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -import os, sys -from git.test.lib import ( - TestBase, - patch, +import os +from git.test.lib import ( TestBase, + patch, raises, assert_equal, assert_true, @@ -65,7 +64,7 @@ def test_it_accepts_stdin(self): @patch.object(Git, 'execute') def test_it_ignores_false_kwargs(self, git): # this_should_not_be_ignored=False implies it *should* be ignored - output = self.git.version(pass_this_kwarg=False) + self.git.version(pass_this_kwarg=False) assert_true("pass_this_kwarg" not in git.call_args[1]) def test_persistent_cat_file_command(self): @@ -87,8 +86,8 @@ def test_persistent_cat_file_command(self): # read data - have to read it in one large chunk size = int(obj_info.split()[2]) data = g.stdout.read(size) - terminating_newline = g.stdout.read(1) - + g.stdout.read(1) + # now we should be able to read a new object g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() From c5452aa820c0f5c2454642587ff6a3bd6d96eaa1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 15:52:17 +0100 Subject: [PATCH 0329/3719] Prepared release 0.3.2 It represents the latest state on github, which should be better than what's installed by default. [skip ci] --- .gitignore | 1 + VERSION | 2 +- doc/source/changes.rst | 6 ++ etc/sublime-text/git-python.sublime-project | 64 ++++++++++----------- git/ext/gitdb | 2 +- requirements.txt | 2 + setup.py | 28 +++++++-- 7 files changed, 66 insertions(+), 39 deletions(-) create mode 100644 requirements.txt mode change 100644 => 100755 setup.py diff --git a/.gitignore b/.gitignore index 1a26c03a1..2e8e17497 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.py[co] *.swp *~ +/*.egg-info /lib/GitPython.egg-info cover/ .coverage diff --git a/VERSION b/VERSION index 5a311b4fc..d15723fbe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.2 RC1 +0.3.2 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index c1e65195c..927f326c7 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +0.3.2 +===== + +* Release of most recent version as non-RC build, just to allow pip to install the latest version right away. +* Have a look at the milestones (https://github.com/gitpython-developers/GitPython/milestones) to see what's next. + 0.3.2 RC1 ========= * **git** command wrapper diff --git a/etc/sublime-text/git-python.sublime-project b/etc/sublime-text/git-python.sublime-project index 5d981925a..d3b692892 100644 --- a/etc/sublime-text/git-python.sublime-project +++ b/etc/sublime-text/git-python.sublime-project @@ -35,37 +35,37 @@ "gitdb/ext" ] }, - // SMMAP - //////// - { - "follow_symlinks": true, - "path": "../../git/ext/gitdb/gitdb/ext/smmap", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - ] - }, - // ASYNC - //////// - { - "follow_symlinks": true, - "path": "../../git/ext/gitdb/gitdb/ext/async", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - ] - }, + // // SMMAP + // //////// + // { + // "follow_symlinks": true, + // "path": "../../git/ext/gitdb/gitdb/ext/smmap", + // "file_exclude_patterns" : [ + // "*.sublime-workspace", + // ".git", + // ".noseids", + // ".coverage" + // ], + // "folder_exclude_patterns" : [ + // ".git", + // "cover", + // ] + // }, + // // ASYNC + // //////// + // { + // "follow_symlinks": true, + // "path": "../../git/ext/gitdb/gitdb/ext/async", + // "file_exclude_patterns" : [ + // "*.sublime-workspace", + // ".git", + // ".noseids", + // ".coverage" + // ], + // "folder_exclude_patterns" : [ + // ".git", + // "cover", + // ] + // }, ] } diff --git a/git/ext/gitdb b/git/ext/gitdb index 39de11274..2f2fe4eea 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 39de1127459b73b862f2b779bb4565ad6b4bd625 +Subproject commit 2f2fe4eea8ba4f47e63a7392a1f27f74f5ee925d diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..c8a4a4148 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +GitPython +gitdb >= 0.6.0 \ No newline at end of file diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index e7c927b13..ed04a5817 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ def _stamp_version(filename): else: print >> sys.stderr, "WARNING: Couldn't find version line in file %s" % filename +install_requires = ('gitdb >= 0.6.0',) setup(name = "GitPython", cmdclass={'build_py': build_py, 'sdist': sdist}, version = VERSION, @@ -73,18 +74,35 @@ def _stamp_version(filename): package_data = {'git.test' : ['fixtures/*']}, package_dir = {'git':'git'}, license = "BSD License", - install_requires='gitdb >= 0.5.1', + requires=('gitdb (>=0.6.0)', ), + install_requires=install_requires, + test_requirements = ('mock', 'nose') + install_requires, zip_safe=False, long_description = """\ GitPython is a python library used to interact with Git repositories""", - classifiers = [ + classifiers=[ + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", "Development Status :: 4 - Beta", + # "Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", - "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", - "Topic :: Software Development :: Libraries :: Python Modules", - ] + "Programming Language :: Python :: 2.7", + # "Programming Language :: Python :: 3", + # "Programming Language :: Python :: 3.3", + # "Programming Language :: Python :: 3.4", + ] ) From e1ad78eb7494513f6c53f0226fe3cb7df4e67513 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 16:05:43 +0100 Subject: [PATCH 0330/3719] Assure requirements.txt ends up in the distribution as well --- MANIFEST.in | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 89f5b92d0..95b2e883f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE include CHANGES include AUTHORS include README +include requirements.txt graft git/test/fixtures graft git/test/performance diff --git a/requirements.txt b/requirements.txt index c8a4a4148..77af7ff82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ GitPython -gitdb >= 0.6.0 \ No newline at end of file +gitdb>=0.6.0 \ No newline at end of file From 598cd1d7f452e05bfcda98ce9e3c392cf554fe75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 17:13:20 +0100 Subject: [PATCH 0331/3719] Simplified get_user_id() and fixed possible python3 compatiblity issue. Changed motivated by https://github.com/gitpython-developers/GitPython/pull/52 --- git/util.py | 46 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/git/util.py b/git/util.py index 88a72c0cb..30ccfa660 100644 --- a/git/util.py +++ b/git/util.py @@ -10,22 +10,20 @@ import time import stat import shutil -import tempfile import platform +import getpass +# NOTE: Some of the unused imports might be used/imported by others. +# Handle once test-cases are back up and running. from gitdb.util import ( - make_sha, - LockedFD, - file_contents_ro, - LazyMixin, - to_hex_sha, - to_bin_sha - ) - -# Import the user database on unix based systems -if os.name == "posix": - import pwd - + make_sha, + LockedFD, + file_contents_ro, + LazyMixin, + to_hex_sha, + to_bin_sha +) + __all__ = ( "stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', @@ -116,19 +114,8 @@ def assure_directory_exists(path, is_file=False): return False def get_user_id(): - """:return: string identifying the currently active system user as name@node - :note: user can be set with the 'USER' environment variable, usually set on windows - :note: on unix based systems you can use the password database - to get the login name of the effective process user""" - if os.name == "posix": - username = pwd.getpwuid(os.geteuid()).pw_name - else: - ukn = 'UNKNOWN' - username = os.environ.get('USER', os.environ.get('USERNAME', ukn)) - if username == ukn and hasattr(os, 'getlogin'): - username = os.getlogin() - # END get username from login - return "%s@%s" % (username, platform.node()) + """:return: string identifying the currently active system user as name@node""" + return "%s@%s" % (getpass.getuser(), platform.node()) #} END utilities @@ -492,7 +479,7 @@ def _obtain_lock_or_raise(self): try: fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0) os.close(fd) - except OSError,e: + except OSError as e: raise IOError(str(e)) self._owns_lock = True @@ -514,7 +501,7 @@ def _release_lock(self): # on bloody windows, the file needs write permissions to be removable. # Why ... if os.name == 'nt': - os.chmod(lfp, 0777) + os.chmod(lfp, int("0777", 8)) # END handle win32 os.remove(lfp) except OSError: @@ -593,9 +580,6 @@ def __new__(cls, id_attr, prefix=''): def __init__(self, id_attr, prefix=''): self._id_attr = id_attr self._prefix = prefix - if not isinstance(id_attr, basestring): - raise ValueError("First parameter must be a string identifying the name-property. Extend the list after initialization") - # END help debugging ! def __contains__(self, attr): # first try identy match for performance From 3287148a2f99fa66028434ce971b0200271437e7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 17:32:46 +0100 Subject: [PATCH 0332/3719] Fixed premature closing of stdout/stderr streams, which caused plenty of errors. The lines were added in commit b38020ae , and I might consider a patch release soon or get ready with 0.3.3. Lets hope not too many installations will be affected. --- git/cmd.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index bd7d5b924..5425d8ffa 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -106,8 +106,6 @@ def wait(self): :raise GitCommandError: if the return status is not 0""" status = self.proc.wait() - self.proc.stdout.close() - self.proc.stderr.close() if status != 0: raise GitCommandError(self.args, status, self.proc.stderr.read()) # END status handling From 0441fdcbc4ea1ab8ce5455f2352436712f1b30bb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 17:48:12 +0100 Subject: [PATCH 0333/3719] Don't use tuples in setup.py requirement specs [skip ci] See https://github.com/gitpython-developers/GitPython/issues/186 for the motivation of this fix. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 064a6e7dd..e724a1296 100755 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ def _stamp_version(filename): else: print >> sys.stderr, "WARNING: Couldn't find version line in file %s" % filename -install_requires = ('gitdb >= 0.6.0',) +install_requires = ['gitdb >= 0.6.0',] setup(name = "GitPython", cmdclass={'build_py': build_py, 'sdist': sdist}, version = VERSION, @@ -76,9 +76,9 @@ def _stamp_version(filename): package_data = {'git.test' : ['fixtures/*']}, package_dir = {'git':'git'}, license = "BSD License", - requires=('gitdb (>=0.6.0)', ), + requires=['gitdb (>=0.6.0)'], install_requires=install_requires, - test_requirements = ('mock', 'nose') + install_requires, + test_requirements = ['mock', 'nose'] + install_requires, zip_safe=False, long_description = """\ GitPython is a python library used to interact with Git repositories""", From 18fff4d4a28295500acd531a1b97bc3b89fad07e Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Fri, 14 Nov 2014 22:11:26 +0100 Subject: [PATCH 0334/3719] tox commands now have {posargs} as argument When invoking an environement, one might want to pass extra argument to the command. That is done in tox by invoking an env and passing the extra arguments after '--' which are then available as '{posargs}'. Examples: # Reports flake8 error statistics tox -eflake8 -- --statistics # Only run test_util.py tests, printing a line per test: tox -epy27 -- --verbose git/test/test_util.py --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 60bfb1d97..4d8273576 100644 --- a/tox.ini +++ b/tox.ini @@ -2,15 +2,15 @@ envlist = py26,py27,flake8 [testenv] -commands = nosetests +commands = nosetests {posargs} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt [testenv:cover] -commands = nosetests --with-coverage +commands = nosetests --with-coverage {posargs} [testenv:flake8] -commands = flake8 +commands = flake8 {posargs} [testenv:venv] commands = {posargs} From 2ab7ac2397d60f1a71a90bf836543f9e0dcad2d0 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Fri, 14 Nov 2014 22:18:03 +0100 Subject: [PATCH 0335/3719] Lint setup.py Pass flake8 on setup.py. I have left behing the 'line too long' errors though since they are usually controversial. The setup() call has been reindented to save a level of indentation. --- setup.py | 65 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index e724a1296..fd19be5a2 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,9 @@ with open('requirements.txt') as reqs_file: requirements = reqs_file.read().splitlines() + class build_py(_build_py): + def run(self): init = path.join(self.build_lib, 'git', '__init__.py') if path.exists(init): @@ -30,7 +32,8 @@ def run(self): class sdist(_sdist): - def make_release_tree (self, base_dir, files): + + def make_release_tree(self, base_dir, files): _sdist.make_release_tree(self, base_dir, files) orig = path.join('git', '__init__.py') assert path.exists(orig), orig @@ -48,7 +51,7 @@ def _stamp_version(filename): except (IOError, OSError): print >> sys.stderr, "Couldn't find file %s to stamp version" % filename return - #END handle error, usually happens during binary builds + # END handle error, usually happens during binary builds for line in f: if '__version__ =' in line: line = line.replace("'git'", "'%s'" % VERSION) @@ -63,35 +66,37 @@ def _stamp_version(filename): else: print >> sys.stderr, "WARNING: Couldn't find version line in file %s" % filename -install_requires = ['gitdb >= 0.6.0',] -setup(name = "GitPython", - cmdclass={'build_py': build_py, 'sdist': sdist}, - version = VERSION, - description = "Python Git Library", - author = "Sebastian Thiel, Michael Trier", - author_email = "byronimo@gmail.com, mtrier@gmail.com", - url = "http://gitorious.org/projects/git-python/", - packages = find_packages('.'), - py_modules = ['git.'+f[:-3] for f in os.listdir('./git') if f.endswith('.py')], - package_data = {'git.test' : ['fixtures/*']}, - package_dir = {'git':'git'}, - license = "BSD License", - requires=['gitdb (>=0.6.0)'], - install_requires=install_requires, - test_requirements = ['mock', 'nose'] + install_requires, - zip_safe=False, - long_description = """\ +install_requires = ['gitdb >= 0.6.0'] + +setup( + name="GitPython", + cmdclass={'build_py': build_py, 'sdist': sdist}, + version=VERSION, + description="Python Git Library", + author="Sebastian Thiel, Michael Trier", + author_email="byronimo@gmail.com, mtrier@gmail.com", + url="http://gitorious.org/projects/git-python/", + packages=find_packages('.'), + py_modules=['git.'+f[:-3] for f in os.listdir('./git') if f.endswith('.py')], + package_data={'git.test': ['fixtures/*']}, + package_dir={'git': 'git'}, + license="BSD License", + requires=['gitdb (>=0.6.0)'], + install_requires=install_requires, + test_requirements=['mock', 'nose'] + install_requires, + zip_safe=False, + long_description="""\ GitPython is a python library used to interact with Git repositories""", - classifiers=[ - # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", + classifiers=[ + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + # "Development Status :: 1 - Planning", + # "Development Status :: 2 - Pre-Alpha", + # "Development Status :: 3 - Alpha", "Development Status :: 4 - Beta", # "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", + # "Development Status :: 6 - Mature", + # "Development Status :: 7 - Inactive", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", @@ -106,5 +111,5 @@ def _stamp_version(filename): # "Programming Language :: Python :: 3", # "Programming Language :: Python :: 3.3", # "Programming Language :: Python :: 3.4", - ] - ) + ] +) From f5d11b750ecc982541d1f936488248f0b42d75d3 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 20:15:50 +0100 Subject: [PATCH 0336/3719] pep8 linting (whitespaces) W191 indentation contains tabs E221 multiple spaces before operator E222 multiple spaces after operator E225 missing whitespace around operator E271 multiple spaces after keyword W292 no newline at end of file W293 blank line contains whitespace W391 blank line at end of file --- git/__init__.py | 5 +- git/cmd.py | 142 ++++++++--------- git/config.py | 126 ++++++++-------- git/db.py | 14 +- git/diff.py | 99 ++++++------ git/exc.py | 6 +- git/index/__init__.py | 2 +- git/index/base.py | 63 ++++---- git/index/fun.py | 44 +++--- git/index/typ.py | 14 +- git/index/util.py | 2 +- git/objects/__init__.py | 2 +- git/objects/base.py | 47 +++--- git/objects/blob.py | 2 +- git/objects/commit.py | 98 ++++++------ git/objects/fun.py | 44 +++--- git/objects/submodule/base.py | 213 +++++++++++++------------- git/objects/submodule/root.py | 74 ++++----- git/objects/submodule/util.py | 18 +-- git/objects/tag.py | 17 +-- git/objects/tree.py | 66 ++++---- git/objects/util.py | 86 +++++------ git/odict.py | 115 +++++++------- git/refs/head.py | 90 ++++++----- git/refs/log.py | 80 +++++----- git/refs/reference.py | 26 ++-- git/refs/remote.py | 10 +- git/refs/symbolic.py | 178 +++++++++++----------- git/refs/tag.py | 40 ++--- git/remote.py | 178 +++++++++++----------- git/repo/__init__.py | 2 +- git/repo/base.py | 142 ++++++++--------- git/repo/fun.py | 56 +++---- git/test/lib/asserts.py | 6 +- git/test/lib/helper.py | 64 ++++---- git/test/performance/lib.py | 20 +-- git/test/performance/test_commit.py | 22 +-- git/test/performance/test_odb.py | 14 +- git/test/performance/test_streams.py | 44 +++--- git/test/performance/test_utils.py | 40 ++--- git/test/test_actor.py | 4 +- git/test/test_base.py | 24 +-- git/test/test_blob.py | 7 +- git/test/test_commit.py | 86 +++++------ git/test/test_config.py | 30 ++-- git/test/test_db.py | 8 +- git/test/test_diff.py | 24 ++- git/test/test_fun.py | 80 +++++----- git/test/test_git.py | 6 +- git/test/test_index.py | 218 +++++++++++++-------------- git/test/test_reflog.py | 36 ++--- git/test/test_refs.py | 157 ++++++++++--------- git/test/test_remote.py | 164 ++++++++++---------- git/test/test_repo.py | 188 ++++++++++++----------- git/test/test_stats.py | 8 +- git/test/test_submodule.py | 201 ++++++++++++------------ git/test/test_tree.py | 49 +++--- git/test/test_util.py | 57 ++++--- git/util.py | 204 ++++++++++++------------- 59 files changed, 1917 insertions(+), 1945 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 9ea811123..6ccafcbba 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -15,13 +15,13 @@ def _init_externals(): """Initialize external projects by putting them into the path""" sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'gitdb')) - + try: import gitdb except ImportError: raise ImportError("'gitdb' could not be found in your PYTHONPATH") #END verify import - + #} END initialization ################# @@ -51,4 +51,3 @@ def _init_externals(): __all__ = [ name for name, obj in locals().items() if not (name.startswith('_') or inspect.ismodule(obj)) ] - diff --git a/git/cmd.py b/git/cmd.py index a1780de7e..c655cdc80 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -30,13 +30,13 @@ def dashify(string): class Git(LazyMixin): """ The Git class manages communication with the Git binary. - + It provides a convenient interface to calling the Git binary, such as in:: - + g = Git( git_dir ) g.init() # calls 'git init' program rval = g.ls_files() # calls 'git ls-files' program - + ``Debugging`` Set the GIT_PYTHON_TRACE environment variable print each invocation of the command to stdout. @@ -44,35 +44,35 @@ class Git(LazyMixin): """ __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", "_git_options") - + # CONFIGURATION # The size in bytes read from stdout when copying git's output to another stream max_chunk_size = 1024*64 - + git_exec_name = "git" # default that should work on linux and windows git_exec_name_win = "git.cmd" # alternate command name, windows only - + # Enables debugging of GitPython's git commands GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) - + # Provide the full path to the git executable. Otherwise it assumes git is in the path _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" GIT_PYTHON_GIT_EXECUTABLE = os.environ.get(_git_exec_env_var, git_exec_name) - - + + class AutoInterrupt(object): """Kill/Interrupt the stored process instance once this instance goes out of scope. It is used to prevent processes piling up in case iterators stop reading. Besides all attributes are wired through to the contained process object. - + The wait method was overridden to perform automatic status code checking and possibly raise.""" - __slots__= ("proc", "args") - + __slots__ = ("proc", "args") + def __init__(self, proc, args ): self.proc = proc self.args = args - + def __del__(self): self.proc.stdout.close() self.proc.stderr.close() @@ -80,11 +80,11 @@ def __del__(self): # did the process finish already so we have a return code ? if self.proc.poll() is not None: return - + # can be that nothing really exists anymore ... if os is None: return - + # try to kill it try: os.kill(self.proc.pid, 2) # interrupt signal @@ -98,13 +98,13 @@ def __del__(self): # is whether we really want to see all these messages. Its annoying no matter what. call(("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(self.proc.pid)), shell=True) # END exception handling - + def __getattr__(self, attr): return getattr(self.proc, attr) - + def wait(self): """Wait for the process and return its status code. - + :raise GitCommandError: if the return status is not 0""" status = self.proc.wait() if status != 0: @@ -112,7 +112,7 @@ def wait(self): # END status handling return status # END auto interrupt - + class CatFileContentStream(object): """Object representing a sized read-only stream returning the contents of an object. @@ -120,20 +120,20 @@ class CatFileContentStream(object): stream once our sized content region is empty. If not all data is read to the end of the objects's lifetime, we read the rest to assure the underlying stream continues to work""" - + __slots__ = ('_stream', '_nbr', '_size') - + def __init__(self, size, stream): self._stream = stream self._size = size self._nbr = 0 # num bytes read - + # special case: if the object is empty, has null bytes, get the # final newline right away. if size == 0: stream.read(1) # END handle empty streams - + def read(self, size=-1): bytes_left = self._size - self._nbr if bytes_left == 0: @@ -147,17 +147,17 @@ def read(self, size=-1): # END check early depletion data = self._stream.read(size) self._nbr += len(data) - + # check for depletion, read our final byte to make the stream usable by others if self._size - self._nbr == 0: self._stream.read(1) # final newline # END finish reading return data - + def readline(self, size=-1): if self._nbr == self._size: return '' - + # clamp size to lowest allowed value bytes_left = self._size - self._nbr if size > -1: @@ -165,21 +165,21 @@ def readline(self, size=-1): else: size = bytes_left # END handle size - + data = self._stream.readline(size) self._nbr += len(data) - + # handle final byte if self._size - self._nbr == 0: self._stream.read(1) # END finish reading - + return data - + def readlines(self, size=-1): if self._nbr == self._size: return list() - + # leave all additional logic to our readline method, we just check the size out = list() nbr = 0 @@ -195,16 +195,16 @@ def readlines(self, size=-1): # END handle size constraint # END readline loop return out - + def __iter__(self): return self - + def next(self): line = self.readline() if not line: raise StopIteration return line - + def __del__(self): bytes_left = self._size - self._nbr if bytes_left: @@ -212,11 +212,11 @@ def __del__(self): # includes terminating newline self._stream.read(bytes_left + 1) # END handle incomplete read - - + + def __init__(self, working_dir=None): """Initialize this instance with: - + :param working_dir: Git directory we should work in. If None, we always work in the current directory as returned by os.getcwd(). @@ -246,13 +246,13 @@ def _set_cache_(self, attr): else: super(Git, self)._set_cache_(attr) #END handle version info - + @property def working_dir(self): """:return: Git directory we are working on""" return self._working_dir - + @property def version_info(self): """ @@ -301,7 +301,7 @@ def execute(self, command, wrapper that will interrupt the process once it goes out of scope. If you use the command in iterators, you should pass the whole process instance instead of a single stream. - + :param output_stream: If set to a file-like object, data produced by the git command will be output to the given stream directly. @@ -309,25 +309,25 @@ def execute(self, command, always be created with a pipe due to issues with subprocess. This merely is a workaround as data will be copied from the output pipe to the given output stream directly. - + :param subprocess_kwargs: Keyword arguments to be passed to subprocess.Popen. Please note that some of the valid kwargs are already set by this method, the ones you specify may not be the same ones. - + :return: * str(output) if extended_output = False (Default) * tuple(int(status), str(stdout), str(stderr)) if extended_output = True - + if ouput_stream is True, the stdout value will be your output stream: * output_stream if extended_output = False * tuple(int(status), output_stream, str(stderr)) if extended_output = True Note git is executed with LC_MESSAGES="C" to ensure consitent output regardless of system language. - + :raise GitCommandError: - + :note: If you add additional keyword arguments to the signature of this method, you must update the execute_kwargs tuple housed in this module.""" @@ -338,8 +338,8 @@ def execute(self, command, if with_keep_cwd or self._working_dir is None: cwd = os.getcwd() else: - cwd=self._working_dir - + cwd = self._working_dir + # Start the process proc = Popen(command, env={"LC_MESSAGES": "C"}, @@ -347,12 +347,12 @@ def execute(self, command, stdin=istream, stderr=PIPE, stdout=PIPE, - close_fds=(os.name=='posix'),# unsupported on linux + close_fds=(os.name == 'posix'),# unsupported on linux **subprocess_kwargs ) if as_process: return self.AutoInterrupt(proc, command) - + # Wait for the process to return status = 0 stdout_value = '' @@ -426,7 +426,7 @@ def __unpack_args(cls, arg_list): if isinstance(arg_list, unicode): return [arg_list.encode('utf-8')] return [ str(arg_list) ] - + outlist = list() for arg in arg_list: if isinstance(arg_list, (list, tuple)): @@ -488,10 +488,10 @@ def _call_process(self, method, *args, **kwargs): # Prepare the argument list opt_args = self.transform_kwargs(**kwargs) - + ext_args = self.__unpack_args([a for a in args if a is not None]) args = opt_args + ext_args - + def make_call(): call = [self.GIT_PYTHON_GIT_EXECUTABLE] @@ -504,7 +504,7 @@ def make_call(): call.extend(args) return call #END utility to recreate call after changes - + if sys.platform == 'win32': try: try: @@ -516,7 +516,7 @@ def make_call(): #END handle overridden variable type(self).GIT_PYTHON_GIT_EXECUTABLE = self.git_exec_name_win call = [self.GIT_PYTHON_GIT_EXECUTABLE] + list(args) - + try: return self.execute(make_call(), **_kwargs) finally: @@ -532,14 +532,14 @@ def make_call(): else: return self.execute(make_call(), **_kwargs) #END handle windows default installation - + def _parse_object_header(self, header_line): """ :param header_line: type_string size_as_int - + :return: (hex_sha, type_string, size_as_int) - + :raise ValueError: if the header contains indication for an error due to incorrect input sha""" tokens = header_line.split() @@ -550,46 +550,46 @@ def _parse_object_header(self, header_line): raise ValueError("SHA %s could not be resolved, git returned: %r" % (tokens[0], header_line.strip())) # END handle actual return value # END error handling - + if len(tokens[0]) != 40: raise ValueError("Failed to parse header: %r" % header_line) return (tokens[0], tokens[1], int(tokens[2])) - + def __prepare_ref(self, ref): # required for command to separate refs on stdin refstr = str(ref) # could be ref-object if refstr.endswith("\n"): return refstr return refstr + "\n" - + def __get_persistent_cmd(self, attr_name, cmd_name, *args,**kwargs): cur_val = getattr(self, attr_name) if cur_val is not None: return cur_val - + options = { "istream" : PIPE, "as_process" : True } options.update( kwargs ) - + cmd = self._call_process( cmd_name, *args, **options ) setattr(self, attr_name, cmd ) return cmd - + def __get_object_header(self, cmd, ref): cmd.stdin.write(self.__prepare_ref(ref)) cmd.stdin.flush() return self._parse_object_header(cmd.stdout.readline()) - + def get_object_header(self, ref): """ Use this method to quickly examine the type and size of the object behind the given ref. - + :note: The method will only suffer from the costs of command invocation once and reuses the command in subsequent calls. - + :return: (hexsha, type_string, size_as_int)""" cmd = self.__get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) - + def get_object_data(self, ref): """ As get_object_header, but returns object data as well :return: (hexsha, type_string, size_as_int,data_string) @@ -598,7 +598,7 @@ def get_object_data(self, ref): data = stream.read(size) del(stream) return (hexsha, typename, size, data) - + def stream_object_data(self, ref): """As get_object_header, but returns the data as a stream :return: (hexsha, type_string, size_as_int, stream) @@ -607,12 +607,12 @@ def stream_object_data(self, ref): cmd = self.__get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) return (hexsha, typename, size, self.CatFileContentStream(size, cmd.stdout)) - + def clear_cache(self): """Clear all kinds of internal caches to release resources. - + Currently persistent commands will be interrupted. - + :return: self""" self.cat_file_all = None self.cat_file_header = None diff --git a/git/config.py b/git/config.py index 285ade6b7..5ad69c6ac 100644 --- a/git/config.py +++ b/git/config.py @@ -35,16 +35,16 @@ def __new__(metacls, name, bases, clsdict): if name in mutating_methods: method_with_values = set_dirty_and_flush_changes(method_with_values) # END mutating methods handling - + clsdict[name] = method_with_values # END for each name/method pair # END for each base # END if mutating methods configuration is set - + new_type = super(MetaParserBuilder, metacls).__new__(metacls, name, bases, clsdict) return new_type - - + + def needs_values(func): """Returns method assuring we read values (on demand) before we try to access them""" @@ -54,7 +54,7 @@ def assure_data_present(self, *args, **kwargs): # END wrapper method assure_data_present.__name__ = func.__name__ return assure_data_present - + def set_dirty_and_flush_changes(non_const_func): """Return method that checks whether given non constant function may be called. If so, the instance will be set dirty. @@ -66,64 +66,64 @@ def flush_changes(self, *args, **kwargs): # END wrapper method flush_changes.__name__ = non_const_func.__name__ return flush_changes - + class SectionConstraint(object): """Constrains a ConfigParser to only option commands which are constrained to always use the section we have been initialized with. - + It supports all ConfigParser methods that operate on an option""" __slots__ = ("_config", "_section_name") _valid_attrs_ = ("get_value", "set_value", "get", "set", "getint", "getfloat", "getboolean", "has_option", "remove_section", "remove_option", "options") - + def __init__(self, config, section): self._config = config self._section_name = section - + def __getattr__(self, attr): if attr in self._valid_attrs_: return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs) return super(SectionConstraint,self).__getattribute__(attr) - + def _call_config(self, method, *args, **kwargs): """Call the configuration at the given method which must take a section name as first argument""" return getattr(self._config, method)(self._section_name, *args, **kwargs) - + @property def config(self): """return: Configparser instance we constrain""" return self._config - + class GitConfigParser(cp.RawConfigParser, object): """Implements specifics required to read git style configuration files. - + This variation behaves much like the git.config command such that the configuration will be read on demand based on the filepath given during initialization. - + The changes will automatically be written once the instance goes out of scope, but can be triggered manually as well. - + The configuration file will be locked if you intend to change values preventing other instances to write concurrently. - + :note: The config is case-sensitive even when queried, hence section and option names must match perfectly.""" __metaclass__ = MetaParserBuilder - - + + #{ Configuration # The lock type determines the type of lock to use in new configuration readers. # They must be compatible to the LockFile interface. # A suitable alternative would be the BlockingLockFile t_lock = LockFile re_comment = re.compile('^\s*[#;]') - + #} END configuration - + OPTCRE = re.compile( r'\s*(?P\u00B6'). - attr('href', '#' + this.id). - attr('title', _('Permalink to this headline')). - appendTo(this); - }); - $('dt[id]').each(function() { - $('\u00B6'). - attr('href', '#' + this.id). - attr('title', _('Permalink to this definition')). - appendTo(this); - }); - }, - - /** - * workaround a firefox stupidity - */ - fixFirefoxAnchorBug : function() { - if (document.location.hash && $.browser.mozilla) - window.setTimeout(function() { - document.location.href += ''; - }, 10); - }, - - /** - * highlight the search words provided in the url in the text - */ - highlightSearchWords : function() { - var params = $.getQueryParameters(); - var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; - if (terms.length) { - var body = $('div.body'); - window.setTimeout(function() { - $.each(terms, function() { - body.highlightText(this.toLowerCase(), 'highlight'); - }); - }, 10); - $('
  • ' + _('Hide Search Matches') + '
    • ') - .appendTo($('.sidebar .this-page-menu')); - } - }, - - /** - * init the modindex toggle buttons - */ - initModIndex : function() { - var togglers = $('img.toggler').click(function() { - var src = $(this).attr('src'); - var idnum = $(this).attr('id').substr(7); - console.log($('tr.cg-' + idnum).toggle()); - if (src.substr(-9) == 'minus.png') - $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); - else - $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); - }).css('display', ''); - if (DOCUMENTATION_OPTIONS.COLLAPSE_MODINDEX) { - togglers.click(); - } - }, - - /** - * helper function to hide the search marks again - */ - hideSearchWords : function() { - $('.sidebar .this-page-menu li.highlight-link').fadeOut(300); - $('span.highlight').removeClass('highlight'); - }, - - /** - * make the url absolute - */ - makeURL : function(relativeURL) { - return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; - }, - - /** - * get the current relative url - */ - getCurrentURL : function() { - var path = document.location.pathname; - var parts = path.split(/\//); - $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { - if (this == '..') - parts.pop(); - }); - var url = parts.join('/'); - return path.substring(url.lastIndexOf('/') + 1, path.length - 1); - } -}; - -// quick alias for translations -_ = Documentation.gettext; - -$(document).ready(function() { - Documentation.init(); -}); diff --git a/doc/doc_index/0.1/_static/file.png b/doc/doc_index/0.1/_static/file.png deleted file mode 100644 index d18082e397e7e54f20721af768c4c2983258f1b4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 392 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b z3=G`DAk4@xYmNj^kiEpy*OmP$HyOL$D9)yc9|lc|nKf<9@eUiWd>3GuTC!a5vdfWYEazjncPj5ZQX%+1 zt8B*4=d)!cdDz4wr^#OMYfqGz$1LDFF>|#>*O?AGil(WEs?wLLy{Gj2J_@opDm%`dlax3yA*@*N$G&*ukFv>P8+2CBWO(qz zD0k1@kN>hhb1_6`&wrCswzINE(evt-5C1B^STi2@PmdKI;Vst0PQB6!2kdN diff --git a/doc/doc_index/0.1/_static/jquery.js b/doc/doc_index/0.1/_static/jquery.js deleted file mode 100644 index 82b98e1d7..000000000 --- a/doc/doc_index/0.1/_static/jquery.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - * jQuery 1.2.6 - New Wave Javascript - * - * Copyright (c) 2008 John Resig (jquery.com) - * Dual licensed under the MIT (MIT-LICENSE.txt) - * and GPL (GPL-LICENSE.txt) licenses. - * - * $Date: 2008-05-24 14:22:17 -0400 (Sat, 24 May 2008) $ - * $Rev: 5685 $ - */ -(function(){var _jQuery=window.jQuery,_$=window.$;var jQuery=window.jQuery=window.$=function(selector,context){return new jQuery.fn.init(selector,context);};var quickExpr=/^[^<]*(<(.|\s)+>)[^>]*$|^#(\w+)$/,isSimple=/^.[^:#\[\.]*$/,undefined;jQuery.fn=jQuery.prototype={init:function(selector,context){selector=selector||document;if(selector.nodeType){this[0]=selector;this.length=1;return this;}if(typeof selector=="string"){var match=quickExpr.exec(selector);if(match&&(match[1]||!context)){if(match[1])selector=jQuery.clean([match[1]],context);else{var elem=document.getElementById(match[3]);if(elem){if(elem.id!=match[3])return jQuery().find(selector);return jQuery(elem);}selector=[];}}else -return jQuery(context).find(selector);}else if(jQuery.isFunction(selector))return jQuery(document)[jQuery.fn.ready?"ready":"load"](selector);return this.setArray(jQuery.makeArray(selector));},jquery:"1.2.6",size:function(){return this.length;},length:0,get:function(num){return num==undefined?jQuery.makeArray(this):this[num];},pushStack:function(elems){var ret=jQuery(elems);ret.prevObject=this;return ret;},setArray:function(elems){this.length=0;Array.prototype.push.apply(this,elems);return this;},each:function(callback,args){return jQuery.each(this,callback,args);},index:function(elem){var ret=-1;return jQuery.inArray(elem&&elem.jquery?elem[0]:elem,this);},attr:function(name,value,type){var options=name;if(name.constructor==String)if(value===undefined)return this[0]&&jQuery[type||"attr"](this[0],name);else{options={};options[name]=value;}return this.each(function(i){for(name in options)jQuery.attr(type?this.style:this,name,jQuery.prop(this,options[name],type,i,name));});},css:function(key,value){if((key=='width'||key=='height')&&parseFloat(value)<0)value=undefined;return this.attr(key,value,"curCSS");},text:function(text){if(typeof text!="object"&&text!=null)return this.empty().append((this[0]&&this[0].ownerDocument||document).createTextNode(text));var ret="";jQuery.each(text||this,function(){jQuery.each(this.childNodes,function(){if(this.nodeType!=8)ret+=this.nodeType!=1?this.nodeValue:jQuery.fn.text([this]);});});return ret;},wrapAll:function(html){if(this[0])jQuery(html,this[0].ownerDocument).clone().insertBefore(this[0]).map(function(){var elem=this;while(elem.firstChild)elem=elem.firstChild;return elem;}).append(this);return this;},wrapInner:function(html){return this.each(function(){jQuery(this).contents().wrapAll(html);});},wrap:function(html){return this.each(function(){jQuery(this).wrapAll(html);});},append:function(){return this.domManip(arguments,true,false,function(elem){if(this.nodeType==1)this.appendChild(elem);});},prepend:function(){return this.domManip(arguments,true,true,function(elem){if(this.nodeType==1)this.insertBefore(elem,this.firstChild);});},before:function(){return this.domManip(arguments,false,false,function(elem){this.parentNode.insertBefore(elem,this);});},after:function(){return this.domManip(arguments,false,true,function(elem){this.parentNode.insertBefore(elem,this.nextSibling);});},end:function(){return this.prevObject||jQuery([]);},find:function(selector){var elems=jQuery.map(this,function(elem){return jQuery.find(selector,elem);});return this.pushStack(/[^+>] [^+>]/.test(selector)||selector.indexOf("..")>-1?jQuery.unique(elems):elems);},clone:function(events){var ret=this.map(function(){if(jQuery.browser.msie&&!jQuery.isXMLDoc(this)){var clone=this.cloneNode(true),container=document.createElement("div");container.appendChild(clone);return jQuery.clean([container.innerHTML])[0];}else -return this.cloneNode(true);});var clone=ret.find("*").andSelf().each(function(){if(this[expando]!=undefined)this[expando]=null;});if(events===true)this.find("*").andSelf().each(function(i){if(this.nodeType==3)return;var events=jQuery.data(this,"events");for(var type in events)for(var handler in events[type])jQuery.event.add(clone[i],type,events[type][handler],events[type][handler].data);});return ret;},filter:function(selector){return this.pushStack(jQuery.isFunction(selector)&&jQuery.grep(this,function(elem,i){return selector.call(elem,i);})||jQuery.multiFilter(selector,this));},not:function(selector){if(selector.constructor==String)if(isSimple.test(selector))return this.pushStack(jQuery.multiFilter(selector,this,true));else -selector=jQuery.multiFilter(selector,this);var isArrayLike=selector.length&&selector[selector.length-1]!==undefined&&!selector.nodeType;return this.filter(function(){return isArrayLike?jQuery.inArray(this,selector)<0:this!=selector;});},add:function(selector){return this.pushStack(jQuery.unique(jQuery.merge(this.get(),typeof selector=='string'?jQuery(selector):jQuery.makeArray(selector))));},is:function(selector){return!!selector&&jQuery.multiFilter(selector,this).length>0;},hasClass:function(selector){return this.is("."+selector);},val:function(value){if(value==undefined){if(this.length){var elem=this[0];if(jQuery.nodeName(elem,"select")){var index=elem.selectedIndex,values=[],options=elem.options,one=elem.type=="select-one";if(index<0)return null;for(var i=one?index:0,max=one?index+1:options.length;i=0||jQuery.inArray(this.name,value)>=0);else if(jQuery.nodeName(this,"select")){var values=jQuery.makeArray(value);jQuery("option",this).each(function(){this.selected=(jQuery.inArray(this.value,values)>=0||jQuery.inArray(this.text,values)>=0);});if(!values.length)this.selectedIndex=-1;}else -this.value=value;});},html:function(value){return value==undefined?(this[0]?this[0].innerHTML:null):this.empty().append(value);},replaceWith:function(value){return this.after(value).remove();},eq:function(i){return this.slice(i,i+1);},slice:function(){return this.pushStack(Array.prototype.slice.apply(this,arguments));},map:function(callback){return this.pushStack(jQuery.map(this,function(elem,i){return callback.call(elem,i,elem);}));},andSelf:function(){return this.add(this.prevObject);},data:function(key,value){var parts=key.split(".");parts[1]=parts[1]?"."+parts[1]:"";if(value===undefined){var data=this.triggerHandler("getData"+parts[1]+"!",[parts[0]]);if(data===undefined&&this.length)data=jQuery.data(this[0],key);return data===undefined&&parts[1]?this.data(parts[0]):data;}else -return this.trigger("setData"+parts[1]+"!",[parts[0],value]).each(function(){jQuery.data(this,key,value);});},removeData:function(key){return this.each(function(){jQuery.removeData(this,key);});},domManip:function(args,table,reverse,callback){var clone=this.length>1,elems;return this.each(function(){if(!elems){elems=jQuery.clean(args,this.ownerDocument);if(reverse)elems.reverse();}var obj=this;if(table&&jQuery.nodeName(this,"table")&&jQuery.nodeName(elems[0],"tr"))obj=this.getElementsByTagName("tbody")[0]||this.appendChild(this.ownerDocument.createElement("tbody"));var scripts=jQuery([]);jQuery.each(elems,function(){var elem=clone?jQuery(this).clone(true)[0]:this;if(jQuery.nodeName(elem,"script"))scripts=scripts.add(elem);else{if(elem.nodeType==1)scripts=scripts.add(jQuery("script",elem).remove());callback.call(obj,elem);}});scripts.each(evalScript);});}};jQuery.fn.init.prototype=jQuery.fn;function evalScript(i,elem){if(elem.src)jQuery.ajax({url:elem.src,async:false,dataType:"script"});else -jQuery.globalEval(elem.text||elem.textContent||elem.innerHTML||"");if(elem.parentNode)elem.parentNode.removeChild(elem);}function now(){return+new Date;}jQuery.extend=jQuery.fn.extend=function(){var target=arguments[0]||{},i=1,length=arguments.length,deep=false,options;if(target.constructor==Boolean){deep=target;target=arguments[1]||{};i=2;}if(typeof target!="object"&&typeof target!="function")target={};if(length==i){target=this;--i;}for(;i-1;}},swap:function(elem,options,callback){var old={};for(var name in options){old[name]=elem.style[name];elem.style[name]=options[name];}callback.call(elem);for(var name in options)elem.style[name]=old[name];},css:function(elem,name,force){if(name=="width"||name=="height"){var val,props={position:"absolute",visibility:"hidden",display:"block"},which=name=="width"?["Left","Right"]:["Top","Bottom"];function getWH(){val=name=="width"?elem.offsetWidth:elem.offsetHeight;var padding=0,border=0;jQuery.each(which,function(){padding+=parseFloat(jQuery.curCSS(elem,"padding"+this,true))||0;border+=parseFloat(jQuery.curCSS(elem,"border"+this+"Width",true))||0;});val-=Math.round(padding+border);}if(jQuery(elem).is(":visible"))getWH();else -jQuery.swap(elem,props,getWH);return Math.max(0,val);}return jQuery.curCSS(elem,name,force);},curCSS:function(elem,name,force){var ret,style=elem.style;function color(elem){if(!jQuery.browser.safari)return false;var ret=defaultView.getComputedStyle(elem,null);return!ret||ret.getPropertyValue("color")=="";}if(name=="opacity"&&jQuery.browser.msie){ret=jQuery.attr(style,"opacity");return ret==""?"1":ret;}if(jQuery.browser.opera&&name=="display"){var save=style.outline;style.outline="0 solid black";style.outline=save;}if(name.match(/float/i))name=styleFloat;if(!force&&style&&style[name])ret=style[name];else if(defaultView.getComputedStyle){if(name.match(/float/i))name="float";name=name.replace(/([A-Z])/g,"-$1").toLowerCase();var computedStyle=defaultView.getComputedStyle(elem,null);if(computedStyle&&!color(elem))ret=computedStyle.getPropertyValue(name);else{var swap=[],stack=[],a=elem,i=0;for(;a&&color(a);a=a.parentNode)stack.unshift(a);for(;i]*?)\/>/g,function(all,front,tag){return tag.match(/^(abbr|br|col|img|input|link|meta|param|hr|area|embed)$/i)?all:front+">";});var tags=jQuery.trim(elem).toLowerCase(),div=context.createElement("div");var wrap=!tags.indexOf("",""]||!tags.indexOf("",""]||tags.match(/^<(thead|tbody|tfoot|colg|cap)/)&&[1,"","
    "]||!tags.indexOf("",""]||(!tags.indexOf("",""]||!tags.indexOf("",""]||jQuery.browser.msie&&[1,"div
    ","
    "]||[0,"",""];div.innerHTML=wrap[1]+elem+wrap[2];while(wrap[0]--)div=div.lastChild;if(jQuery.browser.msie){var tbody=!tags.indexOf(""&&tags.indexOf("=0;--j)if(jQuery.nodeName(tbody[j],"tbody")&&!tbody[j].childNodes.length)tbody[j].parentNode.removeChild(tbody[j]);if(/^\s/.test(elem))div.insertBefore(context.createTextNode(elem.match(/^\s*/)[0]),div.firstChild);}elem=jQuery.makeArray(div.childNodes);}if(elem.length===0&&(!jQuery.nodeName(elem,"form")&&!jQuery.nodeName(elem,"select")))return;if(elem[0]==undefined||jQuery.nodeName(elem,"form")||elem.options)ret.push(elem);else -ret=jQuery.merge(ret,elem);});return ret;},attr:function(elem,name,value){if(!elem||elem.nodeType==3||elem.nodeType==8)return undefined;var notxml=!jQuery.isXMLDoc(elem),set=value!==undefined,msie=jQuery.browser.msie;name=notxml&&jQuery.props[name]||name;if(elem.tagName){var special=/href|src|style/.test(name);if(name=="selected"&&jQuery.browser.safari)elem.parentNode.selectedIndex;if(name in elem&¬xml&&!special){if(set){if(name=="type"&&jQuery.nodeName(elem,"input")&&elem.parentNode)throw"type property can't be changed";elem[name]=value;}if(jQuery.nodeName(elem,"form")&&elem.getAttributeNode(name))return elem.getAttributeNode(name).nodeValue;return elem[name];}if(msie&¬xml&&name=="style")return jQuery.attr(elem.style,"cssText",value);if(set)elem.setAttribute(name,""+value);var attr=msie&¬xml&&special?elem.getAttribute(name,2):elem.getAttribute(name);return attr===null?undefined:attr;}if(msie&&name=="opacity"){if(set){elem.zoom=1;elem.filter=(elem.filter||"").replace(/alpha\([^)]*\)/,"")+(parseInt(value)+''=="NaN"?"":"alpha(opacity="+value*100+")");}return elem.filter&&elem.filter.indexOf("opacity=")>=0?(parseFloat(elem.filter.match(/opacity=([^)]*)/)[1])/100)+'':"";}name=name.replace(/-([a-z])/ig,function(all,letter){return letter.toUpperCase();});if(set)elem[name]=value;return elem[name];},trim:function(text){return(text||"").replace(/^\s+|\s+$/g,"");},makeArray:function(array){var ret=[];if(array!=null){var i=array.length;if(i==null||array.split||array.setInterval||array.call)ret[0]=array;else -while(i)ret[--i]=array[i];}return ret;},inArray:function(elem,array){for(var i=0,length=array.length;i*",this).remove();while(this.firstChild)this.removeChild(this.firstChild);}},function(name,fn){jQuery.fn[name]=function(){return this.each(fn,arguments);};});jQuery.each(["Height","Width"],function(i,name){var type=name.toLowerCase();jQuery.fn[type]=function(size){return this[0]==window?jQuery.browser.opera&&document.body["client"+name]||jQuery.browser.safari&&window["inner"+name]||document.compatMode=="CSS1Compat"&&document.documentElement["client"+name]||document.body["client"+name]:this[0]==document?Math.max(Math.max(document.body["scroll"+name],document.documentElement["scroll"+name]),Math.max(document.body["offset"+name],document.documentElement["offset"+name])):size==undefined?(this.length?jQuery.css(this[0],type):null):this.css(type,size.constructor==String?size:size+"px");};});function num(elem,prop){return elem[0]&&parseInt(jQuery.curCSS(elem[0],prop,true),10)||0;}var chars=jQuery.browser.safari&&parseInt(jQuery.browser.version)<417?"(?:[\\w*_-]|\\\\.)":"(?:[\\w\u0128-\uFFFF*_-]|\\\\.)",quickChild=new RegExp("^>\\s*("+chars+"+)"),quickID=new RegExp("^("+chars+"+)(#)("+chars+"+)"),quickClass=new RegExp("^([#.]?)("+chars+"*)");jQuery.extend({expr:{"":function(a,i,m){return m[2]=="*"||jQuery.nodeName(a,m[2]);},"#":function(a,i,m){return a.getAttribute("id")==m[2];},":":{lt:function(a,i,m){return im[3]-0;},nth:function(a,i,m){return m[3]-0==i;},eq:function(a,i,m){return m[3]-0==i;},first:function(a,i){return i==0;},last:function(a,i,m,r){return i==r.length-1;},even:function(a,i){return i%2==0;},odd:function(a,i){return i%2;},"first-child":function(a){return a.parentNode.getElementsByTagName("*")[0]==a;},"last-child":function(a){return jQuery.nth(a.parentNode.lastChild,1,"previousSibling")==a;},"only-child":function(a){return!jQuery.nth(a.parentNode.lastChild,2,"previousSibling");},parent:function(a){return a.firstChild;},empty:function(a){return!a.firstChild;},contains:function(a,i,m){return(a.textContent||a.innerText||jQuery(a).text()||"").indexOf(m[3])>=0;},visible:function(a){return"hidden"!=a.type&&jQuery.css(a,"display")!="none"&&jQuery.css(a,"visibility")!="hidden";},hidden:function(a){return"hidden"==a.type||jQuery.css(a,"display")=="none"||jQuery.css(a,"visibility")=="hidden";},enabled:function(a){return!a.disabled;},disabled:function(a){return a.disabled;},checked:function(a){return a.checked;},selected:function(a){return a.selected||jQuery.attr(a,"selected");},text:function(a){return"text"==a.type;},radio:function(a){return"radio"==a.type;},checkbox:function(a){return"checkbox"==a.type;},file:function(a){return"file"==a.type;},password:function(a){return"password"==a.type;},submit:function(a){return"submit"==a.type;},image:function(a){return"image"==a.type;},reset:function(a){return"reset"==a.type;},button:function(a){return"button"==a.type||jQuery.nodeName(a,"button");},input:function(a){return/input|select|textarea|button/i.test(a.nodeName);},has:function(a,i,m){return jQuery.find(m[3],a).length;},header:function(a){return/h\d/i.test(a.nodeName);},animated:function(a){return jQuery.grep(jQuery.timers,function(fn){return a==fn.elem;}).length;}}},parse:[/^(\[) *@?([\w-]+) *([!*$^~=]*) *('?"?)(.*?)\4 *\]/,/^(:)([\w-]+)\("?'?(.*?(\(.*?\))?[^(]*?)"?'?\)/,new RegExp("^([:.#]*)("+chars+"+)")],multiFilter:function(expr,elems,not){var old,cur=[];while(expr&&expr!=old){old=expr;var f=jQuery.filter(expr,elems,not);expr=f.t.replace(/^\s*,\s*/,"");cur=not?elems=f.r:jQuery.merge(cur,f.r);}return cur;},find:function(t,context){if(typeof t!="string")return[t];if(context&&context.nodeType!=1&&context.nodeType!=9)return[];context=context||document;var ret=[context],done=[],last,nodeName;while(t&&last!=t){var r=[];last=t;t=jQuery.trim(t);var foundToken=false,re=quickChild,m=re.exec(t);if(m){nodeName=m[1].toUpperCase();for(var i=0;ret[i];i++)for(var c=ret[i].firstChild;c;c=c.nextSibling)if(c.nodeType==1&&(nodeName=="*"||c.nodeName.toUpperCase()==nodeName))r.push(c);ret=r;t=t.replace(re,"");if(t.indexOf(" ")==0)continue;foundToken=true;}else{re=/^([>+~])\s*(\w*)/i;if((m=re.exec(t))!=null){r=[];var merge={};nodeName=m[2].toUpperCase();m=m[1];for(var j=0,rl=ret.length;j=0;if(!not&&pass||not&&!pass)tmp.push(r[i]);}return tmp;},filter:function(t,r,not){var last;while(t&&t!=last){last=t;var p=jQuery.parse,m;for(var i=0;p[i];i++){m=p[i].exec(t);if(m){t=t.substring(m[0].length);m[2]=m[2].replace(/\\/g,"");break;}}if(!m)break;if(m[1]==":"&&m[2]=="not")r=isSimple.test(m[3])?jQuery.filter(m[3],r,true).r:jQuery(r).not(m[3]);else if(m[1]==".")r=jQuery.classFilter(r,m[2],not);else if(m[1]=="["){var tmp=[],type=m[3];for(var i=0,rl=r.length;i=0)^not)tmp.push(a);}r=tmp;}else if(m[1]==":"&&m[2]=="nth-child"){var merge={},tmp=[],test=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(m[3]=="even"&&"2n"||m[3]=="odd"&&"2n+1"||!/\D/.test(m[3])&&"0n+"+m[3]||m[3]),first=(test[1]+(test[2]||1))-0,last=test[3]-0;for(var i=0,rl=r.length;i=0)add=true;if(add^not)tmp.push(node);}r=tmp;}else{var fn=jQuery.expr[m[1]];if(typeof fn=="object")fn=fn[m[2]];if(typeof fn=="string")fn=eval("false||function(a,i){return "+fn+";}");r=jQuery.grep(r,function(elem,i){return fn(elem,i,m,r);},not);}}return{r:r,t:t};},dir:function(elem,dir){var matched=[],cur=elem[dir];while(cur&&cur!=document){if(cur.nodeType==1)matched.push(cur);cur=cur[dir];}return matched;},nth:function(cur,result,dir,elem){result=result||1;var num=0;for(;cur;cur=cur[dir])if(cur.nodeType==1&&++num==result)break;return cur;},sibling:function(n,elem){var r=[];for(;n;n=n.nextSibling){if(n.nodeType==1&&n!=elem)r.push(n);}return r;}});jQuery.event={add:function(elem,types,handler,data){if(elem.nodeType==3||elem.nodeType==8)return;if(jQuery.browser.msie&&elem.setInterval)elem=window;if(!handler.guid)handler.guid=this.guid++;if(data!=undefined){var fn=handler;handler=this.proxy(fn,function(){return fn.apply(this,arguments);});handler.data=data;}var events=jQuery.data(elem,"events")||jQuery.data(elem,"events",{}),handle=jQuery.data(elem,"handle")||jQuery.data(elem,"handle",function(){if(typeof jQuery!="undefined"&&!jQuery.event.triggered)return jQuery.event.handle.apply(arguments.callee.elem,arguments);});handle.elem=elem;jQuery.each(types.split(/\s+/),function(index,type){var parts=type.split(".");type=parts[0];handler.type=parts[1];var handlers=events[type];if(!handlers){handlers=events[type]={};if(!jQuery.event.special[type]||jQuery.event.special[type].setup.call(elem)===false){if(elem.addEventListener)elem.addEventListener(type,handle,false);else if(elem.attachEvent)elem.attachEvent("on"+type,handle);}}handlers[handler.guid]=handler;jQuery.event.global[type]=true;});elem=null;},guid:1,global:{},remove:function(elem,types,handler){if(elem.nodeType==3||elem.nodeType==8)return;var events=jQuery.data(elem,"events"),ret,index;if(events){if(types==undefined||(typeof types=="string"&&types.charAt(0)=="."))for(var type in events)this.remove(elem,type+(types||""));else{if(types.type){handler=types.handler;types=types.type;}jQuery.each(types.split(/\s+/),function(index,type){var parts=type.split(".");type=parts[0];if(events[type]){if(handler)delete events[type][handler.guid];else -for(handler in events[type])if(!parts[1]||events[type][handler].type==parts[1])delete events[type][handler];for(ret in events[type])break;if(!ret){if(!jQuery.event.special[type]||jQuery.event.special[type].teardown.call(elem)===false){if(elem.removeEventListener)elem.removeEventListener(type,jQuery.data(elem,"handle"),false);else if(elem.detachEvent)elem.detachEvent("on"+type,jQuery.data(elem,"handle"));}ret=null;delete events[type];}}});}for(ret in events)break;if(!ret){var handle=jQuery.data(elem,"handle");if(handle)handle.elem=null;jQuery.removeData(elem,"events");jQuery.removeData(elem,"handle");}}},trigger:function(type,data,elem,donative,extra){data=jQuery.makeArray(data);if(type.indexOf("!")>=0){type=type.slice(0,-1);var exclusive=true;}if(!elem){if(this.global[type])jQuery("*").add([window,document]).trigger(type,data);}else{if(elem.nodeType==3||elem.nodeType==8)return undefined;var val,ret,fn=jQuery.isFunction(elem[type]||null),event=!data[0]||!data[0].preventDefault;if(event){data.unshift({type:type,target:elem,preventDefault:function(){},stopPropagation:function(){},timeStamp:now()});data[0][expando]=true;}data[0].type=type;if(exclusive)data[0].exclusive=true;var handle=jQuery.data(elem,"handle");if(handle)val=handle.apply(elem,data);if((!fn||(jQuery.nodeName(elem,'a')&&type=="click"))&&elem["on"+type]&&elem["on"+type].apply(elem,data)===false)val=false;if(event)data.shift();if(extra&&jQuery.isFunction(extra)){ret=extra.apply(elem,val==null?data:data.concat(val));if(ret!==undefined)val=ret;}if(fn&&donative!==false&&val!==false&&!(jQuery.nodeName(elem,'a')&&type=="click")){this.triggered=true;try{elem[type]();}catch(e){}}this.triggered=false;}return val;},handle:function(event){var val,ret,namespace,all,handlers;event=arguments[0]=jQuery.event.fix(event||window.event);namespace=event.type.split(".");event.type=namespace[0];namespace=namespace[1];all=!namespace&&!event.exclusive;handlers=(jQuery.data(this,"events")||{})[event.type];for(var j in handlers){var handler=handlers[j];if(all||handler.type==namespace){event.handler=handler;event.data=handler.data;ret=handler.apply(this,arguments);if(val!==false)val=ret;if(ret===false){event.preventDefault();event.stopPropagation();}}}return val;},fix:function(event){if(event[expando]==true)return event;var originalEvent=event;event={originalEvent:originalEvent};var props="altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode metaKey newValue originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target timeStamp toElement type view wheelDelta which".split(" ");for(var i=props.length;i;i--)event[props[i]]=originalEvent[props[i]];event[expando]=true;event.preventDefault=function(){if(originalEvent.preventDefault)originalEvent.preventDefault();originalEvent.returnValue=false;};event.stopPropagation=function(){if(originalEvent.stopPropagation)originalEvent.stopPropagation();originalEvent.cancelBubble=true;};event.timeStamp=event.timeStamp||now();if(!event.target)event.target=event.srcElement||document;if(event.target.nodeType==3)event.target=event.target.parentNode;if(!event.relatedTarget&&event.fromElement)event.relatedTarget=event.fromElement==event.target?event.toElement:event.fromElement;if(event.pageX==null&&event.clientX!=null){var doc=document.documentElement,body=document.body;event.pageX=event.clientX+(doc&&doc.scrollLeft||body&&body.scrollLeft||0)-(doc.clientLeft||0);event.pageY=event.clientY+(doc&&doc.scrollTop||body&&body.scrollTop||0)-(doc.clientTop||0);}if(!event.which&&((event.charCode||event.charCode===0)?event.charCode:event.keyCode))event.which=event.charCode||event.keyCode;if(!event.metaKey&&event.ctrlKey)event.metaKey=event.ctrlKey;if(!event.which&&event.button)event.which=(event.button&1?1:(event.button&2?3:(event.button&4?2:0)));return event;},proxy:function(fn,proxy){proxy.guid=fn.guid=fn.guid||proxy.guid||this.guid++;return proxy;},special:{ready:{setup:function(){bindReady();return;},teardown:function(){return;}},mouseenter:{setup:function(){if(jQuery.browser.msie)return false;jQuery(this).bind("mouseover",jQuery.event.special.mouseenter.handler);return true;},teardown:function(){if(jQuery.browser.msie)return false;jQuery(this).unbind("mouseover",jQuery.event.special.mouseenter.handler);return true;},handler:function(event){if(withinElement(event,this))return true;event.type="mouseenter";return jQuery.event.handle.apply(this,arguments);}},mouseleave:{setup:function(){if(jQuery.browser.msie)return false;jQuery(this).bind("mouseout",jQuery.event.special.mouseleave.handler);return true;},teardown:function(){if(jQuery.browser.msie)return false;jQuery(this).unbind("mouseout",jQuery.event.special.mouseleave.handler);return true;},handler:function(event){if(withinElement(event,this))return true;event.type="mouseleave";return jQuery.event.handle.apply(this,arguments);}}}};jQuery.fn.extend({bind:function(type,data,fn){return type=="unload"?this.one(type,data,fn):this.each(function(){jQuery.event.add(this,type,fn||data,fn&&data);});},one:function(type,data,fn){var one=jQuery.event.proxy(fn||data,function(event){jQuery(this).unbind(event,one);return(fn||data).apply(this,arguments);});return this.each(function(){jQuery.event.add(this,type,one,fn&&data);});},unbind:function(type,fn){return this.each(function(){jQuery.event.remove(this,type,fn);});},trigger:function(type,data,fn){return this.each(function(){jQuery.event.trigger(type,data,this,true,fn);});},triggerHandler:function(type,data,fn){return this[0]&&jQuery.event.trigger(type,data,this[0],false,fn);},toggle:function(fn){var args=arguments,i=1;while(i=0){var selector=url.slice(off,url.length);url=url.slice(0,off);}callback=callback||function(){};var type="GET";if(params)if(jQuery.isFunction(params)){callback=params;params=null;}else{params=jQuery.param(params);type="POST";}var self=this;jQuery.ajax({url:url,type:type,dataType:"html",data:params,complete:function(res,status){if(status=="success"||status=="notmodified")self.html(selector?jQuery("
    ").append(res.responseText.replace(//g,"")).find(selector):res.responseText);self.each(callback,[res.responseText,status,res]);}});return this;},serialize:function(){return jQuery.param(this.serializeArray());},serializeArray:function(){return this.map(function(){return jQuery.nodeName(this,"form")?jQuery.makeArray(this.elements):this;}).filter(function(){return this.name&&!this.disabled&&(this.checked||/select|textarea/i.test(this.nodeName)||/text|hidden|password/i.test(this.type));}).map(function(i,elem){var val=jQuery(this).val();return val==null?null:val.constructor==Array?jQuery.map(val,function(val,i){return{name:elem.name,value:val};}):{name:elem.name,value:val};}).get();}});jQuery.each("ajaxStart,ajaxStop,ajaxComplete,ajaxError,ajaxSuccess,ajaxSend".split(","),function(i,o){jQuery.fn[o]=function(f){return this.bind(o,f);};});var jsc=now();jQuery.extend({get:function(url,data,callback,type){if(jQuery.isFunction(data)){callback=data;data=null;}return jQuery.ajax({type:"GET",url:url,data:data,success:callback,dataType:type});},getScript:function(url,callback){return jQuery.get(url,null,callback,"script");},getJSON:function(url,data,callback){return jQuery.get(url,data,callback,"json");},post:function(url,data,callback,type){if(jQuery.isFunction(data)){callback=data;data={};}return jQuery.ajax({type:"POST",url:url,data:data,success:callback,dataType:type});},ajaxSetup:function(settings){jQuery.extend(jQuery.ajaxSettings,settings);},ajaxSettings:{url:location.href,global:true,type:"GET",timeout:0,contentType:"application/x-www-form-urlencoded",processData:true,async:true,data:null,username:null,password:null,accepts:{xml:"application/xml, text/xml",html:"text/html",script:"text/javascript, application/javascript",json:"application/json, text/javascript",text:"text/plain",_default:"*/*"}},lastModified:{},ajax:function(s){s=jQuery.extend(true,s,jQuery.extend(true,{},jQuery.ajaxSettings,s));var jsonp,jsre=/=\?(&|$)/g,status,data,type=s.type.toUpperCase();if(s.data&&s.processData&&typeof s.data!="string")s.data=jQuery.param(s.data);if(s.dataType=="jsonp"){if(type=="GET"){if(!s.url.match(jsre))s.url+=(s.url.match(/\?/)?"&":"?")+(s.jsonp||"callback")+"=?";}else if(!s.data||!s.data.match(jsre))s.data=(s.data?s.data+"&":"")+(s.jsonp||"callback")+"=?";s.dataType="json";}if(s.dataType=="json"&&(s.data&&s.data.match(jsre)||s.url.match(jsre))){jsonp="jsonp"+jsc++;if(s.data)s.data=(s.data+"").replace(jsre,"="+jsonp+"$1");s.url=s.url.replace(jsre,"="+jsonp+"$1");s.dataType="script";window[jsonp]=function(tmp){data=tmp;success();complete();window[jsonp]=undefined;try{delete window[jsonp];}catch(e){}if(head)head.removeChild(script);};}if(s.dataType=="script"&&s.cache==null)s.cache=false;if(s.cache===false&&type=="GET"){var ts=now();var ret=s.url.replace(/(\?|&)_=.*?(&|$)/,"$1_="+ts+"$2");s.url=ret+((ret==s.url)?(s.url.match(/\?/)?"&":"?")+"_="+ts:"");}if(s.data&&type=="GET"){s.url+=(s.url.match(/\?/)?"&":"?")+s.data;s.data=null;}if(s.global&&!jQuery.active++)jQuery.event.trigger("ajaxStart");var remote=/^(?:\w+:)?\/\/([^\/?#]+)/;if(s.dataType=="script"&&type=="GET"&&remote.test(s.url)&&remote.exec(s.url)[1]!=location.host){var head=document.getElementsByTagName("head")[0];var script=document.createElement("script");script.src=s.url;if(s.scriptCharset)script.charset=s.scriptCharset;if(!jsonp){var done=false;script.onload=script.onreadystatechange=function(){if(!done&&(!this.readyState||this.readyState=="loaded"||this.readyState=="complete")){done=true;success();complete();head.removeChild(script);}};}head.appendChild(script);return undefined;}var requestDone=false;var xhr=window.ActiveXObject?new ActiveXObject("Microsoft.XMLHTTP"):new XMLHttpRequest();if(s.username)xhr.open(type,s.url,s.async,s.username,s.password);else -xhr.open(type,s.url,s.async);try{if(s.data)xhr.setRequestHeader("Content-Type",s.contentType);if(s.ifModified)xhr.setRequestHeader("If-Modified-Since",jQuery.lastModified[s.url]||"Thu, 01 Jan 1970 00:00:00 GMT");xhr.setRequestHeader("X-Requested-With","XMLHttpRequest");xhr.setRequestHeader("Accept",s.dataType&&s.accepts[s.dataType]?s.accepts[s.dataType]+", */*":s.accepts._default);}catch(e){}if(s.beforeSend&&s.beforeSend(xhr,s)===false){s.global&&jQuery.active--;xhr.abort();return false;}if(s.global)jQuery.event.trigger("ajaxSend",[xhr,s]);var onreadystatechange=function(isTimeout){if(!requestDone&&xhr&&(xhr.readyState==4||isTimeout=="timeout")){requestDone=true;if(ival){clearInterval(ival);ival=null;}status=isTimeout=="timeout"&&"timeout"||!jQuery.httpSuccess(xhr)&&"error"||s.ifModified&&jQuery.httpNotModified(xhr,s.url)&&"notmodified"||"success";if(status=="success"){try{data=jQuery.httpData(xhr,s.dataType,s.dataFilter);}catch(e){status="parsererror";}}if(status=="success"){var modRes;try{modRes=xhr.getResponseHeader("Last-Modified");}catch(e){}if(s.ifModified&&modRes)jQuery.lastModified[s.url]=modRes;if(!jsonp)success();}else -jQuery.handleError(s,xhr,status);complete();if(s.async)xhr=null;}};if(s.async){var ival=setInterval(onreadystatechange,13);if(s.timeout>0)setTimeout(function(){if(xhr){xhr.abort();if(!requestDone)onreadystatechange("timeout");}},s.timeout);}try{xhr.send(s.data);}catch(e){jQuery.handleError(s,xhr,null,e);}if(!s.async)onreadystatechange();function success(){if(s.success)s.success(data,status);if(s.global)jQuery.event.trigger("ajaxSuccess",[xhr,s]);}function complete(){if(s.complete)s.complete(xhr,status);if(s.global)jQuery.event.trigger("ajaxComplete",[xhr,s]);if(s.global&&!--jQuery.active)jQuery.event.trigger("ajaxStop");}return xhr;},handleError:function(s,xhr,status,e){if(s.error)s.error(xhr,status,e);if(s.global)jQuery.event.trigger("ajaxError",[xhr,s,e]);},active:0,httpSuccess:function(xhr){try{return!xhr.status&&location.protocol=="file:"||(xhr.status>=200&&xhr.status<300)||xhr.status==304||xhr.status==1223||jQuery.browser.safari&&xhr.status==undefined;}catch(e){}return false;},httpNotModified:function(xhr,url){try{var xhrRes=xhr.getResponseHeader("Last-Modified");return xhr.status==304||xhrRes==jQuery.lastModified[url]||jQuery.browser.safari&&xhr.status==undefined;}catch(e){}return false;},httpData:function(xhr,type,filter){var ct=xhr.getResponseHeader("content-type"),xml=type=="xml"||!type&&ct&&ct.indexOf("xml")>=0,data=xml?xhr.responseXML:xhr.responseText;if(xml&&data.documentElement.tagName=="parsererror")throw"parsererror";if(filter)data=filter(data,type);if(type=="script")jQuery.globalEval(data);if(type=="json")data=eval("("+data+")");return data;},param:function(a){var s=[];if(a.constructor==Array||a.jquery)jQuery.each(a,function(){s.push(encodeURIComponent(this.name)+"="+encodeURIComponent(this.value));});else -for(var j in a)if(a[j]&&a[j].constructor==Array)jQuery.each(a[j],function(){s.push(encodeURIComponent(j)+"="+encodeURIComponent(this));});else -s.push(encodeURIComponent(j)+"="+encodeURIComponent(jQuery.isFunction(a[j])?a[j]():a[j]));return s.join("&").replace(/%20/g,"+");}});jQuery.fn.extend({show:function(speed,callback){return speed?this.animate({height:"show",width:"show",opacity:"show"},speed,callback):this.filter(":hidden").each(function(){this.style.display=this.oldblock||"";if(jQuery.css(this,"display")=="none"){var elem=jQuery("<"+this.tagName+" />").appendTo("body");this.style.display=elem.css("display");if(this.style.display=="none")this.style.display="block";elem.remove();}}).end();},hide:function(speed,callback){return speed?this.animate({height:"hide",width:"hide",opacity:"hide"},speed,callback):this.filter(":visible").each(function(){this.oldblock=this.oldblock||jQuery.css(this,"display");this.style.display="none";}).end();},_toggle:jQuery.fn.toggle,toggle:function(fn,fn2){return jQuery.isFunction(fn)&&jQuery.isFunction(fn2)?this._toggle.apply(this,arguments):fn?this.animate({height:"toggle",width:"toggle",opacity:"toggle"},fn,fn2):this.each(function(){jQuery(this)[jQuery(this).is(":hidden")?"show":"hide"]();});},slideDown:function(speed,callback){return this.animate({height:"show"},speed,callback);},slideUp:function(speed,callback){return this.animate({height:"hide"},speed,callback);},slideToggle:function(speed,callback){return this.animate({height:"toggle"},speed,callback);},fadeIn:function(speed,callback){return this.animate({opacity:"show"},speed,callback);},fadeOut:function(speed,callback){return this.animate({opacity:"hide"},speed,callback);},fadeTo:function(speed,to,callback){return this.animate({opacity:to},speed,callback);},animate:function(prop,speed,easing,callback){var optall=jQuery.speed(speed,easing,callback);return this[optall.queue===false?"each":"queue"](function(){if(this.nodeType!=1)return false;var opt=jQuery.extend({},optall),p,hidden=jQuery(this).is(":hidden"),self=this;for(p in prop){if(prop[p]=="hide"&&hidden||prop[p]=="show"&&!hidden)return opt.complete.call(this);if(p=="height"||p=="width"){opt.display=jQuery.css(this,"display");opt.overflow=this.style.overflow;}}if(opt.overflow!=null)this.style.overflow="hidden";opt.curAnim=jQuery.extend({},prop);jQuery.each(prop,function(name,val){var e=new jQuery.fx(self,opt,name);if(/toggle|show|hide/.test(val))e[val=="toggle"?hidden?"show":"hide":val](prop);else{var parts=val.toString().match(/^([+-]=)?([\d+-.]+)(.*)$/),start=e.cur(true)||0;if(parts){var end=parseFloat(parts[2]),unit=parts[3]||"px";if(unit!="px"){self.style[name]=(end||1)+unit;start=((end||1)/e.cur(true))*start;self.style[name]=start+unit;}if(parts[1])end=((parts[1]=="-="?-1:1)*end)+start;e.custom(start,end,unit);}else -e.custom(start,val,"");}});return true;});},queue:function(type,fn){if(jQuery.isFunction(type)||(type&&type.constructor==Array)){fn=type;type="fx";}if(!type||(typeof type=="string"&&!fn))return queue(this[0],type);return this.each(function(){if(fn.constructor==Array)queue(this,type,fn);else{queue(this,type).push(fn);if(queue(this,type).length==1)fn.call(this);}});},stop:function(clearQueue,gotoEnd){var timers=jQuery.timers;if(clearQueue)this.queue([]);this.each(function(){for(var i=timers.length-1;i>=0;i--)if(timers[i].elem==this){if(gotoEnd)timers[i](true);timers.splice(i,1);}});if(!gotoEnd)this.dequeue();return this;}});var queue=function(elem,type,array){if(elem){type=type||"fx";var q=jQuery.data(elem,type+"queue");if(!q||array)q=jQuery.data(elem,type+"queue",jQuery.makeArray(array));}return q;};jQuery.fn.dequeue=function(type){type=type||"fx";return this.each(function(){var q=queue(this,type);q.shift();if(q.length)q[0].call(this);});};jQuery.extend({speed:function(speed,easing,fn){var opt=speed&&speed.constructor==Object?speed:{complete:fn||!fn&&easing||jQuery.isFunction(speed)&&speed,duration:speed,easing:fn&&easing||easing&&easing.constructor!=Function&&easing};opt.duration=(opt.duration&&opt.duration.constructor==Number?opt.duration:jQuery.fx.speeds[opt.duration])||jQuery.fx.speeds.def;opt.old=opt.complete;opt.complete=function(){if(opt.queue!==false)jQuery(this).dequeue();if(jQuery.isFunction(opt.old))opt.old.call(this);};return opt;},easing:{linear:function(p,n,firstNum,diff){return firstNum+diff*p;},swing:function(p,n,firstNum,diff){return((-Math.cos(p*Math.PI)/2)+0.5)*diff+firstNum;}},timers:[],timerId:null,fx:function(elem,options,prop){this.options=options;this.elem=elem;this.prop=prop;if(!options.orig)options.orig={};}});jQuery.fx.prototype={update:function(){if(this.options.step)this.options.step.call(this.elem,this.now,this);(jQuery.fx.step[this.prop]||jQuery.fx.step._default)(this);if(this.prop=="height"||this.prop=="width")this.elem.style.display="block";},cur:function(force){if(this.elem[this.prop]!=null&&this.elem.style[this.prop]==null)return this.elem[this.prop];var r=parseFloat(jQuery.css(this.elem,this.prop,force));return r&&r>-10000?r:parseFloat(jQuery.curCSS(this.elem,this.prop))||0;},custom:function(from,to,unit){this.startTime=now();this.start=from;this.end=to;this.unit=unit||this.unit||"px";this.now=this.start;this.pos=this.state=0;this.update();var self=this;function t(gotoEnd){return self.step(gotoEnd);}t.elem=this.elem;jQuery.timers.push(t);if(jQuery.timerId==null){jQuery.timerId=setInterval(function(){var timers=jQuery.timers;for(var i=0;ithis.options.duration+this.startTime){this.now=this.end;this.pos=this.state=1;this.update();this.options.curAnim[this.prop]=true;var done=true;for(var i in this.options.curAnim)if(this.options.curAnim[i]!==true)done=false;if(done){if(this.options.display!=null){this.elem.style.overflow=this.options.overflow;this.elem.style.display=this.options.display;if(jQuery.css(this.elem,"display")=="none")this.elem.style.display="block";}if(this.options.hide)this.elem.style.display="none";if(this.options.hide||this.options.show)for(var p in this.options.curAnim)jQuery.attr(this.elem.style,p,this.options.orig[p]);}if(done)this.options.complete.call(this.elem);return false;}else{var n=t-this.startTime;this.state=n/this.options.duration;this.pos=jQuery.easing[this.options.easing||(jQuery.easing.swing?"swing":"linear")](this.state,n,0,1,this.options.duration);this.now=this.start+((this.end-this.start)*this.pos);this.update();}return true;}};jQuery.extend(jQuery.fx,{speeds:{slow:600,fast:200,def:400},step:{scrollLeft:function(fx){fx.elem.scrollLeft=fx.now;},scrollTop:function(fx){fx.elem.scrollTop=fx.now;},opacity:function(fx){jQuery.attr(fx.elem.style,"opacity",fx.now);},_default:function(fx){fx.elem.style[fx.prop]=fx.now+fx.unit;}}});jQuery.fn.offset=function(){var left=0,top=0,elem=this[0],results;if(elem)with(jQuery.browser){var parent=elem.parentNode,offsetChild=elem,offsetParent=elem.offsetParent,doc=elem.ownerDocument,safari2=safari&&parseInt(version)<522&&!/adobeair/i.test(userAgent),css=jQuery.curCSS,fixed=css(elem,"position")=="fixed";if(elem.getBoundingClientRect){var box=elem.getBoundingClientRect();add(box.left+Math.max(doc.documentElement.scrollLeft,doc.body.scrollLeft),box.top+Math.max(doc.documentElement.scrollTop,doc.body.scrollTop));add(-doc.documentElement.clientLeft,-doc.documentElement.clientTop);}else{add(elem.offsetLeft,elem.offsetTop);while(offsetParent){add(offsetParent.offsetLeft,offsetParent.offsetTop);if(mozilla&&!/^t(able|d|h)$/i.test(offsetParent.tagName)||safari&&!safari2)border(offsetParent);if(!fixed&&css(offsetParent,"position")=="fixed")fixed=true;offsetChild=/^body$/i.test(offsetParent.tagName)?offsetChild:offsetParent;offsetParent=offsetParent.offsetParent;}while(parent&&parent.tagName&&!/^body|html$/i.test(parent.tagName)){if(!/^inline|table.*$/i.test(css(parent,"display")))add(-parent.scrollLeft,-parent.scrollTop);if(mozilla&&css(parent,"overflow")!="visible")border(parent);parent=parent.parentNode;}if((safari2&&(fixed||css(offsetChild,"position")=="absolute"))||(mozilla&&css(offsetChild,"position")!="absolute"))add(-doc.body.offsetLeft,-doc.body.offsetTop);if(fixed)add(Math.max(doc.documentElement.scrollLeft,doc.body.scrollLeft),Math.max(doc.documentElement.scrollTop,doc.body.scrollTop));}results={top:top,left:left};}function border(elem){add(jQuery.curCSS(elem,"borderLeftWidth",true),jQuery.curCSS(elem,"borderTopWidth",true));}function add(l,t){left+=parseInt(l,10)||0;top+=parseInt(t,10)||0;}return results;};jQuery.fn.extend({position:function(){var left=0,top=0,results;if(this[0]){var offsetParent=this.offsetParent(),offset=this.offset(),parentOffset=/^body|html$/i.test(offsetParent[0].tagName)?{top:0,left:0}:offsetParent.offset();offset.top-=num(this,'marginTop');offset.left-=num(this,'marginLeft');parentOffset.top+=num(offsetParent,'borderTopWidth');parentOffset.left+=num(offsetParent,'borderLeftWidth');results={top:offset.top-parentOffset.top,left:offset.left-parentOffset.left};}return results;},offsetParent:function(){var offsetParent=this[0].offsetParent;while(offsetParent&&(!/^body|html$/i.test(offsetParent.tagName)&&jQuery.css(offsetParent,'position')=='static'))offsetParent=offsetParent.offsetParent;return jQuery(offsetParent);}});jQuery.each(['Left','Top'],function(i,name){var method='scroll'+name;jQuery.fn[method]=function(val){if(!this[0])return;return val!=undefined?this.each(function(){this==window||this==document?window.scrollTo(!i?val:jQuery(window).scrollLeft(),i?val:jQuery(window).scrollTop()):this[method]=val;}):this[0]==window||this[0]==document?self[i?'pageYOffset':'pageXOffset']||jQuery.boxModel&&document.documentElement[method]||document.body[method]:this[0][method];};});jQuery.each(["Height","Width"],function(i,name){var tl=i?"Left":"Top",br=i?"Right":"Bottom";jQuery.fn["inner"+name]=function(){return this[name.toLowerCase()]()+num(this,"padding"+tl)+num(this,"padding"+br);};jQuery.fn["outer"+name]=function(margin){return this["inner"+name]()+num(this,"border"+tl+"Width")+num(this,"border"+br+"Width")+(margin?num(this,"margin"+tl)+num(this,"margin"+br):0);};});})(); \ No newline at end of file diff --git a/doc/doc_index/0.1/_static/minus.png b/doc/doc_index/0.1/_static/minus.png deleted file mode 100644 index da1c5620d10c047525a467a425abe9ff5269cfc2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 199 zcmeAS@N?(olHy`uVBq!ia0vp^+#t-s1SHkYJtzcHoCO|{#XvD(5N2eUHAey{$X?>< z>&kweokM_|(Po{+Q=kw>iEBiObAE1aYF-J$w=>iB1I2R$WLpMkF=>bh=@O1TaS?83{1OVknK< z>&kweokM`jkU7Va11Q8%;u=xnoS&PUnpeW`?aZ|OK(QcC7sn8Z%gHvy&v=;Q4jejg zV8NnAO`-4Z@2~&zopr02WF_WB>pF diff --git a/doc/doc_index/0.1/_static/pygments.css b/doc/doc_index/0.1/_static/pygments.css deleted file mode 100644 index 1f2d2b618..000000000 --- a/doc/doc_index/0.1/_static/pygments.css +++ /dev/null @@ -1,61 +0,0 @@ -.hll { background-color: #ffffcc } -.c { color: #408090; font-style: italic } /* Comment */ -.err { border: 1px solid #FF0000 } /* Error */ -.k { color: #007020; font-weight: bold } /* Keyword */ -.o { color: #666666 } /* Operator */ -.cm { color: #408090; font-style: italic } /* Comment.Multiline */ -.cp { color: #007020 } /* Comment.Preproc */ -.c1 { color: #408090; font-style: italic } /* Comment.Single */ -.cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ -.gd { color: #A00000 } /* Generic.Deleted */ -.ge { font-style: italic } /* Generic.Emph */ -.gr { color: #FF0000 } /* Generic.Error */ -.gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.gi { color: #00A000 } /* Generic.Inserted */ -.go { color: #303030 } /* Generic.Output */ -.gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ -.gs { font-weight: bold } /* Generic.Strong */ -.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.gt { color: #0040D0 } /* Generic.Traceback */ -.kc { color: #007020; font-weight: bold } /* Keyword.Constant */ -.kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ -.kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ -.kp { color: #007020 } /* Keyword.Pseudo */ -.kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ -.kt { color: #902000 } /* Keyword.Type */ -.m { color: #208050 } /* Literal.Number */ -.s { color: #4070a0 } /* Literal.String */ -.na { color: #4070a0 } /* Name.Attribute */ -.nb { color: #007020 } /* Name.Builtin */ -.nc { color: #0e84b5; font-weight: bold } /* Name.Class */ -.no { color: #60add5 } /* Name.Constant */ -.nd { color: #555555; font-weight: bold } /* Name.Decorator */ -.ni { color: #d55537; font-weight: bold } /* Name.Entity */ -.ne { color: #007020 } /* Name.Exception */ -.nf { color: #06287e } /* Name.Function */ -.nl { color: #002070; font-weight: bold } /* Name.Label */ -.nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ -.nt { color: #062873; font-weight: bold } /* Name.Tag */ -.nv { color: #bb60d5 } /* Name.Variable */ -.ow { color: #007020; font-weight: bold } /* Operator.Word */ -.w { color: #bbbbbb } /* Text.Whitespace */ -.mf { color: #208050 } /* Literal.Number.Float */ -.mh { color: #208050 } /* Literal.Number.Hex */ -.mi { color: #208050 } /* Literal.Number.Integer */ -.mo { color: #208050 } /* Literal.Number.Oct */ -.sb { color: #4070a0 } /* Literal.String.Backtick */ -.sc { color: #4070a0 } /* Literal.String.Char */ -.sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ -.s2 { color: #4070a0 } /* Literal.String.Double */ -.se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ -.sh { color: #4070a0 } /* Literal.String.Heredoc */ -.si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ -.sx { color: #c65d09 } /* Literal.String.Other */ -.sr { color: #235388 } /* Literal.String.Regex */ -.s1 { color: #4070a0 } /* Literal.String.Single */ -.ss { color: #517918 } /* Literal.String.Symbol */ -.bp { color: #007020 } /* Name.Builtin.Pseudo */ -.vc { color: #bb60d5 } /* Name.Variable.Class */ -.vg { color: #bb60d5 } /* Name.Variable.Global */ -.vi { color: #bb60d5 } /* Name.Variable.Instance */ -.il { color: #208050 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/doc/doc_index/0.1/_static/searchtools.js b/doc/doc_index/0.1/_static/searchtools.js deleted file mode 100644 index e0226258a..000000000 --- a/doc/doc_index/0.1/_static/searchtools.js +++ /dev/null @@ -1,467 +0,0 @@ -/** - * helper function to return a node containing the - * search summary for a given text. keywords is a list - * of stemmed words, hlwords is the list of normal, unstemmed - * words. the first one is used to find the occurance, the - * latter for highlighting it. - */ - -jQuery.makeSearchSummary = function(text, keywords, hlwords) { - var textLower = text.toLowerCase(); - var start = 0; - $.each(keywords, function() { - var i = textLower.indexOf(this.toLowerCase()); - if (i > -1) - start = i; - }); - start = Math.max(start - 120, 0); - var excerpt = ((start > 0) ? '...' : '') + - $.trim(text.substr(start, 240)) + - ((start + 240 - text.length) ? '...' : ''); - var rv = $('
    ').text(excerpt); - $.each(hlwords, function() { - rv = rv.highlightText(this, 'highlight'); - }); - return rv; -} - -/** - * Porter Stemmer - */ -var PorterStemmer = function() { - - var step2list = { - ational: 'ate', - tional: 'tion', - enci: 'ence', - anci: 'ance', - izer: 'ize', - bli: 'ble', - alli: 'al', - entli: 'ent', - eli: 'e', - ousli: 'ous', - ization: 'ize', - ation: 'ate', - ator: 'ate', - alism: 'al', - iveness: 'ive', - fulness: 'ful', - ousness: 'ous', - aliti: 'al', - iviti: 'ive', - biliti: 'ble', - logi: 'log' - }; - - var step3list = { - icate: 'ic', - ative: '', - alize: 'al', - iciti: 'ic', - ical: 'ic', - ful: '', - ness: '' - }; - - var c = "[^aeiou]"; // consonant - var v = "[aeiouy]"; // vowel - var C = c + "[^aeiouy]*"; // consonant sequence - var V = v + "[aeiou]*"; // vowel sequence - - var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 - var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 - var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 - var s_v = "^(" + C + ")?" + v; // vowel in stem - - this.stemWord = function (w) { - var stem; - var suffix; - var firstch; - var origword = w; - - if (w.length < 3) - return w; - - var re; - var re2; - var re3; - var re4; - - firstch = w.substr(0,1); - if (firstch == "y") - w = firstch.toUpperCase() + w.substr(1); - - // Step 1a - re = /^(.+?)(ss|i)es$/; - re2 = /^(.+?)([^s])s$/; - - if (re.test(w)) - w = w.replace(re,"$1$2"); - else if (re2.test(w)) - w = w.replace(re2,"$1$2"); - - // Step 1b - re = /^(.+?)eed$/; - re2 = /^(.+?)(ed|ing)$/; - if (re.test(w)) { - var fp = re.exec(w); - re = new RegExp(mgr0); - if (re.test(fp[1])) { - re = /.$/; - w = w.replace(re,""); - } - } - else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1]; - re2 = new RegExp(s_v); - if (re2.test(stem)) { - w = stem; - re2 = /(at|bl|iz)$/; - re3 = new RegExp("([^aeiouylsz])\\1$"); - re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); - if (re2.test(w)) - w = w + "e"; - else if (re3.test(w)) { - re = /.$/; - w = w.replace(re,""); - } - else if (re4.test(w)) - w = w + "e"; - } - } - - // Step 1c - re = /^(.+?)y$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(s_v); - if (re.test(stem)) - w = stem + "i"; - } - - // Step 2 - re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) - w = stem + step2list[suffix]; - } - - // Step 3 - re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) - w = stem + step3list[suffix]; - } - - // Step 4 - re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; - re2 = /^(.+?)(s|t)(ion)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - if (re.test(stem)) - w = stem; - } - else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1] + fp[2]; - re2 = new RegExp(mgr1); - if (re2.test(stem)) - w = stem; - } - - // Step 5 - re = /^(.+?)e$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - re2 = new RegExp(meq1); - re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); - if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) - w = stem; - } - re = /ll$/; - re2 = new RegExp(mgr1); - if (re.test(w) && re2.test(w)) { - re = /.$/; - w = w.replace(re,""); - } - - // and turn initial Y back to y - if (firstch == "y") - w = firstch.toLowerCase() + w.substr(1); - return w; - } -} - - -/** - * Search Module - */ -var Search = { - - _index : null, - _queued_query : null, - _pulse_status : -1, - - init : function() { - var params = $.getQueryParameters(); - if (params.q) { - var query = params.q[0]; - $('input[name="q"]')[0].value = query; - this.performSearch(query); - } - }, - - /** - * Sets the index - */ - setIndex : function(index) { - var q; - this._index = index; - if ((q = this._queued_query) !== null) { - this._queued_query = null; - Search.query(q); - } - }, - - hasIndex : function() { - return this._index !== null; - }, - - deferQuery : function(query) { - this._queued_query = query; - }, - - stopPulse : function() { - this._pulse_status = 0; - }, - - startPulse : function() { - if (this._pulse_status >= 0) - return; - function pulse() { - Search._pulse_status = (Search._pulse_status + 1) % 4; - var dotString = ''; - for (var i = 0; i < Search._pulse_status; i++) - dotString += '.'; - Search.dots.text(dotString); - if (Search._pulse_status > -1) - window.setTimeout(pulse, 500); - }; - pulse(); - }, - - /** - * perform a search for something - */ - performSearch : function(query) { - // create the required interface elements - this.out = $('#search-results'); - this.title = $('

    ' + _('Searching') + '

    ').appendTo(this.out); - this.dots = $('').appendTo(this.title); - this.status = $('

    ').appendTo(this.out); - this.output = $('