"""

data = cursor.use_region(offset).buffer()

type_id, uncomp_size, data_rela_offset = pack_object_header_info(data)

total_rela_offset = None # set later, actual offset until data stream begins

delta_info = None

# OFFSET DELTA

if type_id == OFS_DELTA:

i = data_rela_offset

c = ord(data[i])

i += 1

delta_offset = c & 0x7f

while c & 0x80:

c = ord(data[i])

i += 1

delta_offset += 1

delta_offset = (delta_offset << 7) + (c & 0x7f)

# END character loop

delta_info = delta_offset

total_rela_offset = i

# REF DELTA

elif type_id == REF_DELTA:

total_rela_offset = data_rela_offset + 20

delta_info = data[data_rela_offset:total_rela_offset]

# BASE OBJECT

else:

# assume its a base object

total_rela_offset = data_rela_offset

# END handle type id

abs_data_offset = offset + total_rela_offset

if as_stream:

stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size)

if delta_info is None:

return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream)

else:

return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream)

else:

if delta_info is None:

return abs_data_offset, OPackInfo(offset, type_id, uncomp_size)

else:

return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info)

# END handle info

"""Copy a stream as read from read function, zip it, and write the result.

br = 0 # bytes read

bw = 0 # bytes written

want_crc = base_crc is not None

crc = 0

if want_crc:

crc = base_crc

# END initialize crc

while True:

chunk = read(chunk_size)

br += len(chunk)

compressed = zstream.compress(chunk)

bw += len(compressed)

write(compressed) # cannot assume return value

if want_crc:

crc = crc32(compressed, crc)

# END handle crc

if len(chunk) != chunk_size:

break

# END copy loop

compressed = zstream.flush()

bw += len(compressed)

write(compressed)

if want_crc:

crc = crc32(compressed, crc)

# END handle crc

"""Utility to cache index information, allowing to write all information later

__slots__ = '_objs'

def __init__(self):

self._objs = list()

def append(self, binsha, crc, offset):

"""Append one piece of object information"""

self._objs.append((binsha, crc, offset))

def write(self, pack_sha, write):

"""Write the index file using the given write method

# sort for sha1 hash

self._objs.sort(key=lambda o: o[0])

sha_writer = FlexibleSha1Writer(write)

sha_write = sha_writer.write

sha_write(PackIndexFile.index_v2_signature)

sha_write(pack(">L", PackIndexFile.index_version_default))

# fanout

tmplist = list((0,) * 256) # fanout or list with 64 bit offsets

for t in self._objs:

tmplist[ord(t[0][0])] += 1

# END prepare fanout

for i in xrange(255):

v = tmplist[i]

sha_write(pack('>L', v))

tmplist[i + 1] += v

# END write each fanout entry

sha_write(pack('>L', tmplist[255]))

# sha1 ordered

# save calls, that is push them into c

sha_write(''.join(t[0] for t in self._objs))

# crc32

for t in self._objs:

sha_write(pack('>L', t[1] & 0xffffffff))

# END for each crc

tmplist = list()

# offset 32

for t in self._objs:

ofs = t[2]

if ofs > 0x7fffffff:

tmplist.append(ofs)

ofs = 0x80000000 + len(tmplist) - 1

# END hande 64 bit offsets

sha_write(pack('>L', ofs & 0xffffffff))

# END for each offset

# offset 64

for ofs in tmplist:

sha_write(pack(">Q", ofs))

# END for each offset

# trailer

assert(len(pack_sha) == 20)

sha_write(pack_sha)

sha = sha_writer.sha(as_hex=False)

write(sha)

"""A pack index provides offsets into the corresponding pack, allowing to find

# Dont use slots as we dynamically bind functions for each version, need a dict for this

# The slots you see here are just to keep track of our instance variables

# __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version',

# '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset')

# used in v2 indices

_sha_list_offset = 8 + 1024

index_v2_signature = '\377tOc'

index_version_default = 2

def __init__(self, indexpath):

super(PackIndexFile, self).__init__()

self._indexpath = indexpath

def _set_cache_(self, attr):

if attr == "_packfile_checksum":

self._packfile_checksum = self._cursor.map()[-40:-20]

elif attr == "_packfile_checksum":

self._packfile_checksum = self._cursor.map()[-20:]

elif attr == "_cursor":

# Note: We don't lock the file when reading as we cannot be sure

# that we can actually write to the location - it could be a read-only

# alternate for instance

self._cursor = mman.make_cursor(self._indexpath).use_region()

# We will assume that the index will always fully fit into memory !

if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size():

raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (

self._indexpath, self._cursor.file_size(), mman.window_size()))

# END assert window size

else:

# now its time to initialize everything - if we are here, someone wants

# to access the fanout table or related properties

# CHECK VERSION

mmap = self._cursor.map()

self._version = (mmap[:4] == self.index_v2_signature and 2) or 1

if self._version == 2:

version_id = unpack_from(">L", mmap, 4)[0]

assert version_id == self._version, "Unsupported index version: %i" % version_id

# END assert version

# SETUP FUNCTIONS

# setup our functions according to the actual version

for fname in ('entry', 'offset', 'sha', 'crc'):

setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version)))

# END for each function to initialize

# INITIALIZE DATA

# byte offset is 8 if version is 2, 0 otherwise

self._initialize()

# END handle attributes

#{ Access V1

def _entry_v1(self, i):

""":return: tuple(offset, binsha, 0)"""

return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, )

def _offset_v1(self, i):

"""see ``_offset_v2``"""

return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0]

def _sha_v1(self, i):

"""see ``_sha_v2``"""

base = 1024 + (i * 24) + 4

return self._cursor.map()[base:base + 20]

def _crc_v1(self, i):

"""unsupported"""

return 0

#} END access V1

#{ Access V2

def _entry_v2(self, i):

""":return: tuple(offset, binsha, crc)"""

return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i))

def _offset_v2(self, i):

""":return: 32 or 64 byte offset into pack files. 64 byte offsets will only

offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0]

# if the high-bit is set, this indicates that we have to lookup the offset

# in the 64 bit region of the file. The current offset ( lower 31 bits )

# are the index into it

if offset & 0x80000000:

offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0]

# END handle 64 bit offset

return offset

def _sha_v2(self, i):

""":return: sha at the given index of this file index instance"""

base = self._sha_list_offset + i * 20

return self._cursor.map()[base:base + 20]

def _crc_v2(self, i):

""":return: 4 bytes crc for the object at index i"""

return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0]

#} END access V2

#{ Initialization

def _initialize(self):

"""initialize base data"""

self._fanout_table = self._read_fanout((self._version == 2) * 8)

if self._version == 2:

self._crc_list_offset = self._sha_list_offset + self.size() * 20

self._pack_offset = self._crc_list_offset + self.size() * 4

self._pack_64_offset = self._pack_offset + self.size() * 4

# END setup base

def _read_fanout(self, byte_offset):

"""Generate a fanout table from our data"""

d = self._cursor.map()

out = list()

append = out.append

for i in range(256):

append(unpack_from('>L', d, byte_offset + i * 4)[0])

# END for each entry

return out

#} END initialization

#{ Properties

def version(self):

return self._version

def size(self):

""":return: amount of objects referred to by this index"""

return self._fanout_table[255]

def path(self):

""":return: path to the packindexfile"""

return self._indexpath

def packfile_checksum(self):

""":return: 20 byte sha representing the sha1 hash of the pack file"""

return self._cursor.map()[-40:-20]

def indexfile_checksum(self):

""":return: 20 byte sha representing the sha1 hash of this index file"""

return self._cursor.map()[-20:]

def offsets(self):

""":return: sequence of all offsets in the order in which they were written

if self._version == 2:

# read stream to array, convert to tuple

a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears

a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset))

# networkbyteorder to something array likes more

if sys.byteorder == 'little':

a.byteswap()

return a

else:

return tuple(self.offset(index) for index in xrange(self.size()))

# END handle version

def sha_to_index(self, sha):

"""

first_byte = ord(sha[0])

get_sha = self.sha

lo = 0 # lower index, the left bound of the bisection

if first_byte != 0:

lo = self._fanout_table[first_byte - 1]

hi = self._fanout_table[first_byte] # the upper, right bound of the bisection

# bisect until we have the sha

while lo < hi:

mid = (lo + hi) / 2

c = cmp(sha, get_sha(mid))

if c < 0:

hi = mid

elif not c:

return mid

else:

lo = mid + 1

# END handle midpoint

# END bisect

return None

def partial_sha_to_index(self, partial_bin_sha, canonical_length):

"""

if len(partial_bin_sha) < 2:

raise ValueError("Require at least 2 bytes of partial sha")

first_byte = ord(partial_bin_sha[0])

get_sha = self.sha

lo = 0 # lower index, the left bound of the bisection

if first_byte != 0:

lo = self._fanout_table[first_byte - 1]

hi = self._fanout_table[first_byte] # the upper, right bound of the bisection

# fill the partial to full 20 bytes

filled_sha = partial_bin_sha + '\0' * (20 - len(partial_bin_sha))

# find lowest

while lo < hi:

mid = (lo + hi) / 2

c = cmp(filled_sha, get_sha(mid))

if c < 0:

hi = mid

elif not c:

# perfect match

lo = mid

break

else:

lo = mid + 1

# END handle midpoint

# END bisect

if lo < self.size():

cur_sha = get_sha(lo)

if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha):

next_sha = None

if lo + 1 < self.size():

next_sha = get_sha(lo + 1)

if next_sha and next_sha == cur_sha:

raise AmbiguousObjectName(partial_bin_sha)

return lo

# END if we have a match

# END if we found something

return None

if 'PackIndexFile_sha_to_index' in globals():

# NOTE: Its just about 25% faster, the major bottleneck might be the attr

# accesses

def sha_to_index(self, sha):

return PackIndexFile_sha_to_index(self, sha)

# END redefine heavy-hitter with c version

"""A pack is a file written according to the Version 2 for git packs

__slots__ = ('_packpath', '_cursor', '_size', '_version')

pack_signature = 0x5041434b # 'PACK'

pack_version_default = 2

# offset into our data at which the first object starts

first_object_offset = 3 * 4 # header bytes

footer_size = 20 # final sha

def __init__(self, packpath):

self._packpath = packpath

def _set_cache_(self, attr):

# we fill the whole cache, whichever attribute gets queried first

self._cursor = mman.make_cursor(self._packpath).use_region()

# read the header information

type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0)

# TODO: figure out whether we should better keep the lock, or maybe

# add a .keep file instead ?

if type_id != self.pack_signature:

raise ParseError("Invalid pack signature: %i" % type_id)

def _iter_objects(self, start_offset, as_stream=True):

"""Handle the actual iteration of objects within this pack"""

c = self._cursor

content_size = c.file_size() - self.footer_size

cur_offset = start_offset or self.first_object_offset

null = NullStream()

while cur_offset < content_size:

data_offset, ostream = pack_object_at(c, cur_offset, True)

# scrub the stream to the end - this decompresses the object, but yields

# the amount of compressed bytes we need to get to the next offset

stream_copy(ostream.read, null.write, ostream.size, chunk_size)

cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read()

# if a stream is requested, reset it beforehand

# Otherwise return the Stream object directly, its derived from the

# info object

if as_stream:

ostream.stream.seek(0)

yield ostream

# END until we have read everything

#{ Pack Information

def size(self):

""":return: The amount of objects stored in this pack"""

return self._size

def version(self):

""":return: the version of this pack"""

return self._version

def data(self):

"""

# can use map as we are starting at offset 0. Otherwise we would have to use buffer()

return self._cursor.use_region().map()

def checksum(self):

""":return: 20 byte sha1 hash on all object sha's contained in this file"""

return self._cursor.use_region(self._cursor.file_size() - 20).buffer()[:]

def path(self):

""":return: path to the packfile"""

return self._packpath

#} END pack information

#{ Pack Specific

def collect_streams(self, offset):

"""

out = list()

c = self._cursor

while True:

ostream = pack_object_at(c, offset, True)[1]

out.append(ostream)

if ostream.type_id == OFS_DELTA:

offset = ostream.pack_offset - ostream.delta_info

else:

# the only thing we can lookup are OFFSET deltas. Everything

# else is either an object, or a ref delta, in the latter

# case someone else has to find it

break

# END handle type

# END while chaining streams

return out

#} END pack specific

#{ Read-Database like Interface

def info(self, offset):

"""Retrieve information about the object at the given file-absolute offset

return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1]

def stream(self, offset):

"""Retrieve an object at the given file-relative offset as stream along with its information

return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1]

def stream_iter(self, start_offset=0):

"""

return self._iter_objects(start_offset, as_stream=True)

"""Combines the PackIndexFile and the PackFile into one, allowing the

__slots__ = ('_index', # our index file

'_pack', # our pack file

'_offset_map' # on demand dict mapping one offset to the next consecutive one

)

IndexFileCls = PackIndexFile

PackFileCls = PackFile

def __init__(self, pack_or_index_path):

"""Initialize ourselves with the path to the respective pack or index file"""

basename, ext = os.path.splitext(pack_or_index_path)

self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance

self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance

def _set_cache_(self, attr):

# currently this can only be _offset_map

# TODO: make this a simple sorted offset array which can be bisected

# to find the respective entry, from which we can take a +1 easily

# This might be slower, but should also be much lighter in memory !

offsets_sorted = sorted(self._index.offsets())

last_offset = len(self._pack.data()) - self._pack.footer_size

assert offsets_sorted, "Cannot handle empty indices"

offset_map = None

if len(offsets_sorted) == 1:

offset_map = {offsets_sorted[0]: last_offset}

else:

iter_offsets = iter(offsets_sorted)

iter_offsets_plus_one = iter(offsets_sorted)

iter_offsets_plus_one.next()

consecutive = izip(iter_offsets, iter_offsets_plus_one)

offset_map = dict(consecutive)

# the last offset is not yet set

offset_map[offsets_sorted[-1]] = last_offset

# END handle offset amount

self._offset_map = offset_map

def _sha_to_index(self, sha):

""":return: index for the given sha, or raise"""

index = self._index.sha_to_index(sha)

if index is None:

raise BadObject(sha)

return index

def _iter_objects(self, as_stream):

"""Iterate over all objects in our index and yield their OInfo or OStream instences"""

_sha = self._index.sha

_object = self._object

for index in xrange(self._index.size()):

yield _object(_sha(index), as_stream, index)

# END for each index

def _object(self, sha, as_stream, index=-1):

""":return: OInfo or OStream object providing information about the given sha

# its a little bit redundant here, but it needs to be efficient

if index < 0:

index = self._sha_to_index(sha)

if sha is None:

sha = self._index.sha(index)

# END assure sha is present ( in output )

offset = self._index.offset(index)

type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer())

if as_stream:

if type_id not in delta_types:

packstream = self._pack.stream(offset)

return OStream(sha, packstream.type, packstream.size, packstream.stream)

# END handle non-deltas

# produce a delta stream containing all info

# To prevent it from applying the deltas when querying the size,

# we extract it from the delta stream ourselves

streams = self.collect_streams_at_offset(offset)

dstream = DeltaApplyReader.new(streams)

return ODeltaStream(sha, dstream.type, None, dstream)

else:

if type_id not in delta_types:

return OInfo(sha, type_id_to_type_map[type_id], uncomp_size)

# END handle non-deltas

# deltas are a little tougher - unpack the first bytes to obtain

# the actual target size, as opposed to the size of the delta data

streams = self.collect_streams_at_offset(offset)

buf = streams[0].read(512)

offset, src_size = msb_size(buf)

offset, target_size = msb_size(buf, offset)

# collect the streams to obtain the actual object type

if streams[-1].type_id in delta_types:

raise BadObject(sha, "Could not resolve delta object")

return OInfo(sha, streams[-1].type, target_size)

# END handle stream

#{ Read-Database like Interface

def info(self, sha):

"""Retrieve information about the object identified by the given sha

return self._object(sha, False)

def stream(self, sha):

"""Retrieve an object stream along with its information as identified by the given sha

return self._object(sha, True)

def info_at_index(self, index):

"""As ``info``, but uses a PackIndexFile compatible index to refer to the object"""

return self._object(None, False, index)

def stream_at_index(self, index):

"""As ``stream``, but uses a PackIndexFile compatible index to refer to the

return self._object(None, True, index)

#} END Read-Database like Interface

#{ Interface

def pack(self):

""":return: the underlying pack file instance"""

return self._pack

def index(self):

""":return: the underlying pack index file instance"""

return self._index

def is_valid_stream(self, sha, use_crc=False):

"""

if use_crc:

if self._index.version() < 2:

raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead")

# END handle index version

index = self._sha_to_index(sha)

offset = self._index.offset(index)

next_offset = self._offset_map[offset]

crc_value = self._index.crc(index)

# create the current crc value, on the compressed object data

# Read it in chunks, without copying the data

crc_update = zlib.crc32

pack_data = self._pack.data()

cur_pos = offset

this_crc_value = 0

while cur_pos < next_offset:

rbound = min(cur_pos + chunk_size, next_offset)

size = rbound - cur_pos

this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value)

cur_pos += size

# END window size loop

# crc returns signed 32 bit numbers, the AND op forces it into unsigned

# mode ... wow, sneaky, from dulwich.

return (this_crc_value & 0xffffffff) == crc_value

else:

shawriter = Sha1Writer()

stream = self._object(sha, as_stream=True)

# write a loose object, which is the basis for the sha

write_object(stream.type, stream.size, stream.read, shawriter.write)

assert shawriter.sha(as_hex=False) == sha

return shawriter.sha(as_hex=False) == sha

# END handle crc/sha verification

return True

def info_iter(self):

"""

return self._iter_objects(as_stream=False)

def stream_iter(self):

"""

return self._iter_objects(as_stream=True)

def collect_streams_at_offset(self, offset):

"""

streams = self._pack.collect_streams(offset)

# try to resolve the last one if needed. It is assumed to be either

# a REF delta, or a base object, as OFFSET deltas are resolved by the pack

if streams[-1].type_id == REF_DELTA:

stream = streams[-1]

while stream.type_id in delta_types:

if stream.type_id == REF_DELTA:

sindex = self._index.sha_to_index(stream.delta_info)

if sindex is None:

break

stream = self._pack.stream(self._index.offset(sindex))

streams.append(stream)

else:

# must be another OFS DELTA - this could happen if a REF

# delta we resolve previously points to an OFS delta. Who

# would do that ;) ? We can handle it though

stream = self._pack.stream(stream.delta_info)

streams.append(stream)

# END handle ref delta

# END resolve ref streams

# END resolve streams

return streams

def collect_streams(self, sha):

"""

return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha)))

@classmethod

def write_pack(cls, object_iter, pack_write, index_write=None,

object_count=None, zlib_compression=zlib.Z_BEST_SPEED):

"""

objs = object_iter

if not object_count:

if not isinstance(object_iter, (tuple, list)):

objs = list(object_iter)

# END handle list type

object_count = len(objs)

# END handle object

pack_writer = FlexibleSha1Writer(pack_write)

pwrite = pack_writer.write

ofs = 0 # current offset into the pack file

index = None

wants_index = index_write is not None

# write header

pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count))

ofs += 12

if wants_index:

index = IndexWriter()

# END handle index header

actual_count = 0

for obj in objs:

actual_count += 1

crc = 0

# object header

hdr = create_pack_object_header(obj.type_id, obj.size)

if index_write:

crc = crc32(hdr)

else:

crc = None

# END handle crc

pwrite(hdr)

# data stream

zstream = zlib.compressobj(zlib_compression)

ostream = obj.stream

br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc=crc)

assert(br == obj.size)

if wants_index:

index.append(obj.binsha, crc, ofs)

# END handle index

ofs += len(hdr) + bw

if actual_count == object_count:

break

# END abort once we are done

# END for each object

if actual_count != object_count:

raise ValueError(

"Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count))

# END count assertion

# write footer

pack_sha = pack_writer.sha(as_hex=False)

assert len(pack_sha) == 20

pack_write(pack_sha)

ofs += len(pack_sha) # just for completeness ;)

index_sha = None

if wants_index:

index_sha = index.write(pack_sha, index_write)

# END handle index

return pack_sha, index_sha

@classmethod

def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.Z_BEST_SPEED):

"""Create a new on-disk entity comprised of a properly named pack file and a properly named

pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir)

index_fd, index_path = tempfile.mkstemp('', 'index', base_dir)

pack_write = lambda d: os.write(pack_fd, d)

index_write = lambda d: os.write(index_fd, d)

pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression)

os.close(pack_fd)

os.close(index_fd)

fmt = "pack-%s.%s"

new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack'))

new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx'))