Merge pull request #512 from mcuadros/idx-64bits
format: idxfile, support for >2Gb packfiles
diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go
index 4243f76..f361213 100644
--- a/plumbing/format/idxfile/decoder.go
+++ b/plumbing/format/idxfile/decoder.go
@@ -123,6 +123,7 @@
func readOffsets(idx *Idxfile, r io.Reader) error {
c := int(idx.ObjectCount)
+
for i := 0; i < c; i++ {
o, err := binary.ReadUint32(r)
if err != nil {
@@ -132,6 +133,19 @@
idx.Entries[i].Offset = uint64(o)
}
+ for i := 0; i < c; i++ {
+ if idx.Entries[i].Offset <= offsetLimit {
+ continue
+ }
+
+ o, err := binary.ReadUint64(r)
+ if err != nil {
+ return err
+ }
+
+ idx.Entries[i].Offset = o
+ }
+
return nil
}
diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go
index 991232d..c7decb2 100644
--- a/plumbing/format/idxfile/decoder_test.go
+++ b/plumbing/format/idxfile/decoder_test.go
@@ -2,6 +2,7 @@
import (
"bytes"
+ "encoding/base64"
"fmt"
"testing"
@@ -65,3 +66,76 @@
c.Assert(idx.Entries, DeepEquals, i.Entries)
}
+
+func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) {
+ f := bytes.NewBufferString(fixtureLarge4GB)
+
+ idx := &Idxfile{}
+
+ d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+ err := d.Decode(idx)
+ c.Assert(err, IsNil)
+
+ expected := map[string]uint64{
+ "303953e5aa461c203a324821bc1717f9b4fff895": 12,
+ "5296768e3d9f661387ccbff18c4dea6c997fd78c": 142,
+ "03fc8d58d44267274edef4585eaeeb445879d33f": 1601322837,
+ "8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772": 2646996529,
+ "e0d1d625010087f79c9e01ad9d8f95e1628dda02": 3452385606,
+ "90eba326cdc4d1d61c5ad25224ccbf08731dd041": 3707047470,
+ "bab53055add7bc35882758a922c54a874d6b1272": 5323223332,
+ "1b8995f51987d8a449ca5ea4356595102dc2fbd4": 5894072943,
+ "35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919,
+ }
+
+ for _, e := range idx.Entries {
+ c.Assert(expected[e.Hash.String()], Equals, e.Offset)
+ }
+}
+
+func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) {
+ f := bytes.NewBufferString(fixtureLarge4GB)
+
+ expected := &Idxfile{}
+
+ d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+ err := d.Decode(expected)
+ c.Assert(err, IsNil)
+
+ buf := bytes.NewBuffer(nil)
+ _, err = NewEncoder(buf).Encode(expected)
+ c.Assert(err, IsNil)
+
+ idx := &Idxfile{}
+ err = NewDecoder(buf).Decode(idx)
+ c.Assert(err, IsNil)
+
+ c.Assert(idx.Entries, DeepEquals, expected.Entries)
+}
+
+const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA
+AAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAA
+AAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAA
+AgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAQAAAAE
+AAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQA
+AAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABQAA
+AAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAA
+BQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAF
+AAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUA
+AAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAA
+AAUAAAAFAAAABQAAAAYAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAA
+BwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAH
+AAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcA
+AAAHAAAABwAAAAcAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAA
+AAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAA
+CAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAkAAAAJ
+AAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkA
+AAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAA
+AAkAAAAJA/yNWNRCZydO3vRYXq7rRFh50z8biZX1GYfYpEnKXqQ1ZZUQLcL71DA5U+WqRhwgOjJI
+IbwXF/m0//iVNYWL6cb1kUy+Z2hInEHraAmivOtSlnaOPZ9mE4fMv/GMTepsmX/XjI88606ky55K
+D3UXletByaTwe+dykOujJs3E0dYcWtJSJMy/CHMd0EG6tTBVrde8NYgnWKkixUqHTWsScuDR1iUB
+AIf3nJ4BrZ2PleFijdoCkp36qiGHwFa8NHxMnInZ0s3CKEKmHe+KcZPzuqwmm44GvqGAX3I/VYAA
+AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA
+ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp
+ch2xUA==
+`
diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go
index d8f4d94..40abfb8 100644
--- a/plumbing/format/idxfile/encoder.go
+++ b/plumbing/format/idxfile/encoder.go
@@ -98,13 +98,28 @@
func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) {
sz := 0
+
+ var o64bits []uint64
for _, ent := range idx.Entries {
- if err := binary.WriteUint32(e, uint32(ent.Offset)); err != nil {
+ o := ent.Offset
+ if o > offsetLimit {
+ o64bits = append(o64bits, o)
+ o = offsetLimit + uint64(len(o64bits))
+ }
+
+ if err := binary.WriteUint32(e, uint32(o)); err != nil {
return sz, err
}
sz += 4
+ }
+ for _, o := range o64bits {
+ if err := binary.WriteUint64(e, o); err != nil {
+ return sz, err
+ }
+
+ sz += 8
}
return sz, nil
diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go
index b9bb1c2..6b05eaa 100644
--- a/plumbing/format/idxfile/idxfile.go
+++ b/plumbing/format/idxfile/idxfile.go
@@ -5,6 +5,8 @@
const (
// VersionSupported is the only idx version supported.
VersionSupported = 2
+
+ offsetLimit = 0x7fffffff
)
var (
diff --git a/utils/binary/read.go b/utils/binary/read.go
index c256ffe..50da1ff 100644
--- a/utils/binary/read.go
+++ b/utils/binary/read.go
@@ -94,7 +94,17 @@
lengthBits = uint8(7) // subsequent bytes has 7 bits to store the length
)
-// ReadUint32 reads 4 bytes and returns them as a Big ndian uint32
+// ReadUint64 reads 8 bytes and returns them as a BigEndian uint32
+func ReadUint64(r io.Reader) (uint64, error) {
+ var v uint64
+ if err := binary.Read(r, binary.BigEndian, &v); err != nil {
+ return 0, err
+ }
+
+ return v, nil
+}
+
+// ReadUint32 reads 4 bytes and returns them as a BigEndian uint32
func ReadUint32(r io.Reader) (uint32, error) {
var v uint32
if err := binary.Read(r, binary.BigEndian, &v); err != nil {
diff --git a/utils/binary/write.go b/utils/binary/write.go
index 2ec3581..c08c73a 100644
--- a/utils/binary/write.go
+++ b/utils/binary/write.go
@@ -31,6 +31,12 @@
return err
}
+// WriteUint64 writes the binary representation of a uint64 into w, in BigEndian
+// order
+func WriteUint64(w io.Writer, value uint64) error {
+ return binary.Write(w, binary.BigEndian, value)
+}
+
// WriteUint32 writes the binary representation of a uint32 into w, in BigEndian
// order
func WriteUint32(w io.Writer, value uint32) error {
