Add support for compressed images when pre-loading images (#2165)

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/docker/docker v1.4.2-0.20191205034852-d163fbba3c82
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
+	github.com/frankban/quicktest v1.10.2 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gogo/googleapis v1.3.0 // indirect
@@ -93,6 +94,7 @@ require (
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/opencontainers/runc v1.0.0-rc10
 	github.com/opencontainers/selinux v1.5.3-0.20200613095409-bb88c45a3863
+	github.com/pierrec/lz4 v2.5.2+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/rancher/dynamiclistener v0.2.1
 	github.com/rancher/helm-controller v0.7.2

go.sum

@@ -215,6 +215,8 @@ github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
+github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -366,6 +368,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.0.0-20190617215043-876b8855d23c/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -475,6 +479,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJ
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@@ -603,6 +609,8 @@ github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
+github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -995,6 +1003,8 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191017205301-920acffc3e65 h1:GwXwgmbrvlcHLDsENMqrQTTIC2C0kIPszsq929NruKI=
 golang.org/x/tools v0.0.0-20191017205301-920acffc3e65/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q=

pkg/agent/containerd/containerd.go

@@ -2,6 +2,8 @@ package containerd
 
 import (
 	"bufio"
+	"compress/bzip2"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -16,6 +18,7 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/natefinch/lumberjack"
 	"github.com/opencontainers/runc/libcontainer/system"
+	"github.com/pierrec/lz4"
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/agent/templates"
 	util2 "github.com/rancher/k3s/pkg/agent/util"
@@ -182,7 +185,24 @@ func preloadImages(ctx context.Context, cfg *config.Node) error {
 		}
 
 		logrus.Debugf("Import %s", filePath)
-		_, err = client.Import(ctxContainerD, file)
+		var imageReader io.Reader
+		imageReader = file
+		if strings.HasSuffix(fileInfo.Name(), ".tar.bz2") {
+			imageReader = bzip2.NewReader(file)
+		}
+		if strings.HasSuffix(fileInfo.Name(), ".tar.lz4") {
+			imageReader = lz4.NewReader(file)
+		}
+		if strings.HasSuffix(fileInfo.Name(), ".tar.gz") {
+			// WARNING: gzip reader close does not close the underlying image
+			imageReader, err = gzip.NewReader(file)
+			if err != nil {
+				logrus.Errorf("Unable to import %s: %v", filePath, err)
+				file.Close()
+				continue
+			}
+		}
+		_, err = client.Import(ctxContainerD, imageReader)
 		file.Close()
 		if err != nil {
 			logrus.Errorf("Unable to import %s: %v", filePath, err)

vendor/github.com/google/go-cmp/cmp/export_panic.go

@@ -8,8 +8,8 @@ package cmp
 
 import "reflect"
 
-const supportAllowUnexported = false
+const supportExporters = false
 
-func retrieveUnexportedField(reflect.Value, reflect.StructField) reflect.Value {
-	panic("retrieveUnexportedField is not implemented")
+func retrieveUnexportedField(reflect.Value, reflect.StructField, bool) reflect.Value {
+	panic("no support for forcibly accessing unexported fields")
 }

vendor/github.com/google/go-cmp/cmp/export_unsafe.go

@@ -11,13 +11,25 @@ import (
 	"unsafe"
 )
 
-const supportAllowUnexported = true
+const supportExporters = true
 
 // retrieveUnexportedField uses unsafe to forcibly retrieve any field from
 // a struct such that the value has read-write permissions.
 //
 // The parent struct, v, must be addressable, while f must be a StructField
-// describing the field to retrieve.
-func retrieveUnexportedField(v reflect.Value, f reflect.StructField) reflect.Value {
-	return reflect.NewAt(f.Type, unsafe.Pointer(v.UnsafeAddr()+f.Offset)).Elem()
+// describing the field to retrieve. If addr is false,
+// then the returned value will be shallowed copied to be non-addressable.
+func retrieveUnexportedField(v reflect.Value, f reflect.StructField, addr bool) reflect.Value {
+	ve := reflect.NewAt(f.Type, unsafe.Pointer(uintptr(unsafe.Pointer(v.UnsafeAddr()))+f.Offset)).Elem()
+	if !addr {
+		// A field is addressable if and only if the struct is addressable.
+		// If the original parent value was not addressable, shallow copy the
+		// value to make it non-addressable to avoid leaking an implementation
+		// detail of how forcibly exporting a field works.
+		if ve.Kind() == reflect.Interface && ve.IsNil() {
+			return reflect.Zero(f.Type)
+		}
+		return reflect.ValueOf(ve.Interface()).Convert(f.Type)
+	}
+	return ve
 }

vendor/github.com/google/go-cmp/cmp/internal/diff/diff.go

@@ -12,6 +12,13 @@
 // is more important than obtaining a minimal Levenshtein distance.
 package diff
 
+import (
+	"math/rand"
+	"time"
+
+	"github.com/google/go-cmp/cmp/internal/flags"
+)
+
 // EditType represents a single operation within an edit-script.
 type EditType uint8
 
@@ -112,6 +119,8 @@ func (r Result) Similar() bool {
 	return r.NumSame+1 >= r.NumDiff
 }
 
+var randInt = rand.New(rand.NewSource(time.Now().Unix())).Intn(2)
+
 // Difference reports whether two lists of lengths nx and ny are equal
 // given the definition of equality provided as f.
 //
@@ -159,6 +168,17 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 	// A vertical edge is equivalent to inserting a symbol from list Y.
 	// A diagonal edge is equivalent to a matching symbol between both X and Y.
 
+	// To ensure flexibility in changing the algorithm in the future,
+	// introduce some degree of deliberate instability.
+	// This is achieved by fiddling the zigzag iterator to start searching
+	// the graph starting from the bottom-right versus than the top-left.
+	// The result may differ depending on the starting search location,
+	// but still produces a valid edit script.
+	zigzagInit := randInt // either 0 or 1
+	if flags.Deterministic {
+		zigzagInit = 0
+	}
+
 	// Invariants:
 	//	• 0 ≤ fwdPath.X ≤ (fwdFrontier.X, revFrontier.X) ≤ revPath.X ≤ nx
 	//	• 0 ≤ fwdPath.Y ≤ (fwdFrontier.Y, revFrontier.Y) ≤ revPath.Y ≤ ny
@@ -209,7 +229,7 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 		if fwdFrontier.X >= revFrontier.X || fwdFrontier.Y >= revFrontier.Y || searchBudget == 0 {
 			break
 		}
-		for stop1, stop2, i := false, false, 0; !(stop1 && stop2) && searchBudget > 0; i++ {
+		for stop1, stop2, i := false, false, zigzagInit; !(stop1 && stop2) && searchBudget > 0; i++ {
 			// Search in a diagonal pattern for a match.
 			z := zigzag(i)
 			p := point{fwdFrontier.X + z, fwdFrontier.Y - z}

vendor/github.com/google/go-cmp/cmp/internal/value/name.go

+// Copyright 2020, The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package value
+
+import (
+	"reflect"
+	"strconv"
+)
+
+// TypeString is nearly identical to reflect.Type.String,
+// but has an additional option to specify that full type names be used.
+func TypeString(t reflect.Type, qualified bool) string {
+	return string(appendTypeName(nil, t, qualified, false))
+}
+
+func appendTypeName(b []byte, t reflect.Type, qualified, elideFunc bool) []byte {
+	// BUG: Go reflection provides no way to disambiguate two named types
+	// of the same name and within the same package,
+	// but declared within the namespace of different functions.
+
+	// Named type.
+	if t.Name() != "" {
+		if qualified && t.PkgPath() != "" {
+			b = append(b, '"')
+			b = append(b, t.PkgPath()...)
+			b = append(b, '"')
+			b = append(b, '.')
+			b = append(b, t.Name()...)
+		} else {
+			b = append(b, t.String()...)
+		}
+		return b
+	}
+
+	// Unnamed type.
+	switch k := t.Kind(); k {
+	case reflect.Bool, reflect.String, reflect.UnsafePointer,
+		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+		reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
+		b = append(b, k.String()...)
+	case reflect.Chan:
+		if t.ChanDir() == reflect.RecvDir {
+			b = append(b, "<-"...)
+		}
+		b = append(b, "chan"...)
+		if t.ChanDir() == reflect.SendDir {
+			b = append(b, "<-"...)
+		}
+		b = append(b, ' ')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Func:
+		if !elideFunc {
+			b = append(b, "func"...)
+		}
+		b = append(b, '(')
+		for i := 0; i < t.NumIn(); i++ {
+			if i > 0 {
+				b = append(b, ", "...)
+			}
+			if i == t.NumIn()-1 && t.IsVariadic() {
+				b = append(b, "..."...)
+				b = appendTypeName(b, t.In(i).Elem(), qualified, false)
+			} else {
+				b = appendTypeName(b, t.In(i), qualified, false)
+			}
+		}
+		b = append(b, ')')
+		switch t.NumOut() {
+		case 0:
+			// Do nothing
+		case 1:
+			b = append(b, ' ')
+			b = appendTypeName(b, t.Out(0), qualified, false)
+		default:
+			b = append(b, " ("...)
+			for i := 0; i < t.NumOut(); i++ {
+				if i > 0 {
+					b = append(b, ", "...)
+				}
+				b = appendTypeName(b, t.Out(i), qualified, false)
+			}
+			b = append(b, ')')
+		}
+	case reflect.Struct:
+		b = append(b, "struct{ "...)
+		for i := 0; i < t.NumField(); i++ {
+			if i > 0 {
+				b = append(b, "; "...)
+			}
+			sf := t.Field(i)
+			if !sf.Anonymous {
+				if qualified && sf.PkgPath != "" {
+					b = append(b, '"')
+					b = append(b, sf.PkgPath...)
+					b = append(b, '"')
+					b = append(b, '.')
+				}
+				b = append(b, sf.Name...)
+				b = append(b, ' ')
+			}
+			b = appendTypeName(b, sf.Type, qualified, false)
+			if sf.Tag != "" {
+				b = append(b, ' ')
+				b = strconv.AppendQuote(b, string(sf.Tag))
+			}
+		}
+		if b[len(b)-1] == ' ' {
+			b = b[:len(b)-1]
+		} else {
+			b = append(b, ' ')
+		}
+		b = append(b, '}')
+	case reflect.Slice, reflect.Array:
+		b = append(b, '[')
+		if k == reflect.Array {
+			b = strconv.AppendUint(b, uint64(t.Len()), 10)
+		}
+		b = append(b, ']')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Map:
+		b = append(b, "map["...)
+		b = appendTypeName(b, t.Key(), qualified, false)
+		b = append(b, ']')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Ptr:
+		b = append(b, '*')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Interface:
+		b = append(b, "interface{ "...)
+		for i := 0; i < t.NumMethod(); i++ {
+			if i > 0 {
+				b = append(b, "; "...)
+			}
+			m := t.Method(i)
+			if qualified && m.PkgPath != "" {
+				b = append(b, '"')
+				b = append(b, m.PkgPath...)
+				b = append(b, '"')
+				b = append(b, '.')
+			}
+			b = append(b, m.Name...)
+			b = appendTypeName(b, m.Type, qualified, true)
+		}
+		if b[len(b)-1] == ' ' {
+			b = b[:len(b)-1]
+		} else {
+			b = append(b, ' ')
+		}
+		b = append(b, '}')
+	default:
+		panic("invalid kind: " + k.String())
+	}
+	return b
+}

vendor/github.com/google/go-cmp/cmp/internal/value/pointer_purego.go

@@ -21,3 +21,13 @@ func PointerOf(v reflect.Value) Pointer {
 	// assumes that the GC implementation does not use a moving collector.
 	return Pointer{v.Pointer(), v.Type()}
 }
+
+// IsNil reports whether the pointer is nil.
+func (p Pointer) IsNil() bool {
+	return p.p == 0
+}
+
+// Uintptr returns the pointer as a uintptr.
+func (p Pointer) Uintptr() uintptr {
+	return p.p
+}

vendor/github.com/google/go-cmp/cmp/internal/value/pointer_unsafe.go

@@ -24,3 +24,13 @@ func PointerOf(v reflect.Value) Pointer {
 	// which is necessary if the GC ever uses a moving collector.
 	return Pointer{unsafe.Pointer(v.Pointer()), v.Type()}
 }
+
+// IsNil reports whether the pointer is nil.
+func (p Pointer) IsNil() bool {
+	return p.p == nil
+}
+
+// Uintptr returns the pointer as a uintptr.
+func (p Pointer) Uintptr() uintptr {
+	return uintptr(p.p)
+}

vendor/github.com/google/go-cmp/cmp/options.go

@@ -225,8 +225,23 @@ func (validator) apply(s *state, vx, vy reflect.Value) {
 
 	// Unable to Interface implies unexported field without visibility access.
 	if !vx.CanInterface() || !vy.CanInterface() {
-		const help = "consider using a custom Comparer; if you control the implementation of type, you can also consider AllowUnexported or cmpopts.IgnoreUnexported"
-		panic(fmt.Sprintf("cannot handle unexported field: %#v\n%s", s.curPath, help))
+		help := "consider using a custom Comparer; if you control the implementation of type, you can also consider using an Exporter, AllowUnexported, or cmpopts.IgnoreUnexported"
+		var name string
+		if t := s.curPath.Index(-2).Type(); t.Name() != "" {
+			// Named type with unexported fields.
+			name = fmt.Sprintf("%q.%v", t.PkgPath(), t.Name()) // e.g., "path/to/package".MyType
+			if _, ok := reflect.New(t).Interface().(error); ok {
+				help = "consider using cmpopts.EquateErrors to compare error values"
+			}
+		} else {
+			// Unnamed type with unexported fields. Derive PkgPath from field.
+			var pkgPath string
+			for i := 0; i < t.NumField() && pkgPath == ""; i++ {
+				pkgPath = t.Field(i).PkgPath
+			}
+			name = fmt.Sprintf("%q.(%v)", pkgPath, t.String()) // e.g., "path/to/package".(struct { a int })
+		}
+		panic(fmt.Sprintf("cannot handle unexported field at %#v:\n\t%v\n%s", s.curPath, name, help))
 	}
 
 	panic("not reachable")
@@ -360,9 +375,8 @@ func (cm comparer) String() string {
 	return fmt.Sprintf("Comparer(%s)", function.NameOf(cm.fnc))
 }
 
-// AllowUnexported returns an Option that forcibly allows operations on
-// unexported fields in certain structs, which are specified by passing in a
-// value of each struct type.
+// Exporter returns an Option that specifies whether Equal is allowed to
+// introspect into the unexported fields of certain struct types.
 //
 // Users of this option must understand that comparing on unexported fields
 // from external packages is not safe since changes in the internal
@@ -386,10 +400,24 @@ func (cm comparer) String() string {
 //
 // In other cases, the cmpopts.IgnoreUnexported option can be used to ignore
 // all unexported fields on specified struct types.
-func AllowUnexported(types ...interface{}) Option {
-	if !supportAllowUnexported {
-		panic("AllowUnexported is not supported on purego builds, Google App Engine Standard, or GopherJS")
+func Exporter(f func(reflect.Type) bool) Option {
+	if !supportExporters {
+		panic("Exporter is not supported on purego builds")
 	}
+	return exporter(f)
+}
+
+type exporter func(reflect.Type) bool
+
+func (exporter) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
+	panic("not implemented")
+}
+
+// AllowUnexported returns an Options that allows Equal to forcibly introspect
+// unexported fields of the specified struct types.
+//
+// See Exporter for the proper use of this option.
+func AllowUnexported(types ...interface{}) Option {
 	m := make(map[reflect.Type]bool)
 	for _, typ := range types {
 		t := reflect.TypeOf(typ)
@@ -398,13 +426,7 @@ func AllowUnexported(types ...interface{}) Option {
 		}
 		m[t] = true
 	}
-	return visibleStructs(m)
-}
-
-type visibleStructs map[reflect.Type]bool
-
-func (visibleStructs) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
-	panic("not implemented")
+	return exporter(func(t reflect.Type) bool { return m[t] })
 }
 
 // Result represents the comparison result for a single node and
@@ -436,6 +458,11 @@ func (r Result) ByFunc() bool {
 	return r.flags&reportByFunc != 0
 }
 
+// ByCycle reports whether a reference cycle was detected.
+func (r Result) ByCycle() bool {
+	return r.flags&reportByCycle != 0
+}
+
 type resultFlags uint
 
 const (
@@ -446,6 +473,7 @@ const (
 	reportByIgnore
 	reportByMethod
 	reportByFunc
+	reportByCycle
 )
 
 // Reporter is an Option that can be passed to Equal. When Equal traverses

vendor/github.com/google/go-cmp/cmp/path.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"unicode"
 	"unicode/utf8"
+
+	"github.com/google/go-cmp/cmp/internal/value"
 )
 
 // Path is a list of PathSteps describing the sequence of operations to get
@@ -41,7 +43,7 @@ type PathStep interface {
 	// In some cases, one or both may be invalid or have restrictions:
 	//	• For StructField, both are not interface-able if the current field
 	//	is unexported and the struct type is not explicitly permitted by
-	//	AllowUnexported to traverse unexported fields.
+	//	an Exporter to traverse unexported fields.
 	//	• For SliceIndex, one may be invalid if an element is missing from
 	//	either the x or y slice.
 	//	• For MapIndex, one may be invalid if an entry is missing from
@@ -175,7 +177,8 @@ type structField struct {
 	// pvx, pvy, and field are only valid if unexported is true.
 	unexported bool
 	mayForce   bool                // Forcibly allow visibility
-	pvx, pvy   reflect.Value       // Parent values
+	paddr      bool                // Was parent addressable?
+	pvx, pvy   reflect.Value       // Parent values (always addressible)
 	field      reflect.StructField // Field information
 }
 
@@ -187,8 +190,8 @@ func (sf StructField) Values() (vx, vy reflect.Value) {
 
 	// Forcibly obtain read-write access to an unexported struct field.
 	if sf.mayForce {
-		vx = retrieveUnexportedField(sf.pvx, sf.field)
-		vy = retrieveUnexportedField(sf.pvy, sf.field)
+		vx = retrieveUnexportedField(sf.pvx, sf.field, sf.paddr)
+		vy = retrieveUnexportedField(sf.pvy, sf.field, sf.paddr)
 		return vx, vy // CanInterface reports true
 	}
 	return sf.vx, sf.vy // CanInterface reports false
@@ -207,6 +210,7 @@ type SliceIndex struct{ *sliceIndex }
 type sliceIndex struct {
 	pathStep
 	xkey, ykey int
+	isSlice    bool // False for reflect.Array
 }
 
 func (si SliceIndex) Type() reflect.Type             { return si.typ }
@@ -301,6 +305,72 @@ func (tf Transform) Func() reflect.Value { return tf.trans.fnc }
 // The == operator can be used to detect the exact option used.
 func (tf Transform) Option() Option { return tf.trans }
 
+// pointerPath represents a dual-stack of pointers encountered when
+// recursively traversing the x and y values. This data structure supports
+// detection of cycles and determining whether the cycles are equal.
+// In Go, cycles can occur via pointers, slices, and maps.
+//
+// The pointerPath uses a map to represent a stack; where descension into a
+// pointer pushes the address onto the stack, and ascension from a pointer
+// pops the address from the stack. Thus, when traversing into a pointer from
+// reflect.Ptr, reflect.Slice element, or reflect.Map, we can detect cycles
+// by checking whether the pointer has already been visited. The cycle detection
+// uses a seperate stack for the x and y values.
+//
+// If a cycle is detected we need to determine whether the two pointers
+// should be considered equal. The definition of equality chosen by Equal
+// requires two graphs to have the same structure. To determine this, both the
+// x and y values must have a cycle where the previous pointers were also
+// encountered together as a pair.
+//
+// Semantically, this is equivalent to augmenting Indirect, SliceIndex, and
+// MapIndex with pointer information for the x and y values.
+// Suppose px and py are two pointers to compare, we then search the
+// Path for whether px was ever encountered in the Path history of x, and
+// similarly so with py. If either side has a cycle, the comparison is only
+// equal if both px and py have a cycle resulting from the same PathStep.
+//
+// Using a map as a stack is more performant as we can perform cycle detection
+// in O(1) instead of O(N) where N is len(Path).
+type pointerPath struct {
+	// mx is keyed by x pointers, where the value is the associated y pointer.
+	mx map[value.Pointer]value.Pointer
+	// my is keyed by y pointers, where the value is the associated x pointer.
+	my map[value.Pointer]value.Pointer
+}
+
+func (p *pointerPath) Init() {
+	p.mx = make(map[value.Pointer]value.Pointer)
+	p.my = make(map[value.Pointer]value.Pointer)
+}
+
+// Push indicates intent to descend into pointers vx and vy where
+// visited reports whether either has been seen before. If visited before,
+// equal reports whether both pointers were encountered together.
+// Pop must be called if and only if the pointers were never visited.
+//
+// The pointers vx and vy must be a reflect.Ptr, reflect.Slice, or reflect.Map
+// and be non-nil.
+func (p pointerPath) Push(vx, vy reflect.Value) (equal, visited bool) {
+	px := value.PointerOf(vx)
+	py := value.PointerOf(vy)
+	_, ok1 := p.mx[px]
+	_, ok2 := p.my[py]
+	if ok1 || ok2 {
+		equal = p.mx[px] == py && p.my[py] == px // Pointers paired together
+		return equal, true
+	}
+	p.mx[px] = py
+	p.my[py] = px
+	return false, false
+}
+
+// Pop ascends from pointers vx and vy.
+func (p pointerPath) Pop(vx, vy reflect.Value) {
+	delete(p.mx, value.PointerOf(vx))
+	delete(p.my, value.PointerOf(vy))
+}
+
 // isExported reports whether the identifier is exported.
 func isExported(id string) bool {
 	r, _ := utf8.DecodeRuneInString(id)

vendor/github.com/google/go-cmp/cmp/report.go

@@ -41,7 +41,10 @@ func (r *defaultReporter) String() string {
 	if r.root.NumDiff == 0 {
 		return ""
 	}
-	return formatOptions{}.FormatDiff(r.root).String()
+	ptrs := new(pointerReferences)
+	text := formatOptions{}.FormatDiff(r.root, ptrs)
+	resolveReferences(text)
+	return text.String()
 }
 
 func assert(ok bool) {

vendor/github.com/google/go-cmp/cmp/report_references.go

+// Copyright 2020, The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package cmp
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/google/go-cmp/cmp/internal/flags"
+	"github.com/google/go-cmp/cmp/internal/value"
+)
+
+const (
+	pointerDelimPrefix = "⟪"
+	pointerDelimSuffix = "⟫"
+)
+
+// formatPointer prints the address of the pointer.
+func formatPointer(p value.Pointer, withDelims bool) string {
+	v := p.Uintptr()
+	if flags.Deterministic {
+		v = 0xdeadf00f // Only used for stable testing purposes
+	}
+	if withDelims {
+		return pointerDelimPrefix + formatHex(uint64(v)) + pointerDelimSuffix
+	}
+	return formatHex(uint64(v))
+}
+
+// pointerReferences is a stack of pointers visited so far.
+type pointerReferences [][2]value.Pointer
+
+func (ps *pointerReferences) PushPair(vx, vy reflect.Value, d diffMode, deref bool) (pp [2]value.Pointer) {
+	if deref && vx.IsValid() {
+		vx = vx.Addr()
+	}
+	if deref && vy.IsValid() {
+		vy = vy.Addr()
+	}
+	switch d {
+	case diffUnknown, diffIdentical:
+		pp = [2]value.Pointer{value.PointerOf(vx), value.PointerOf(vy)}
+	case diffRemoved:
+		pp = [2]value.Pointer{value.PointerOf(vx), value.Pointer{}}
+	case diffInserted:
+		pp = [2]value.Pointer{value.Pointer{}, value.PointerOf(vy)}
+	}
+	*ps = append(*ps, pp)
+	return pp
+}
+
+func (ps *pointerReferences) Push(v reflect.Value) (p value.Pointer, seen bool) {
+	p = value.PointerOf(v)
+	for _, pp := range *ps {
+		if p == pp[0] || p == pp[1] {
+			return p, true
+		}
+	}
+	*ps = append(*ps, [2]value.Pointer{p, p})
+	return p, false
+}
+
+func (ps *pointerReferences) Pop() {
+	*ps = (*ps)[:len(*ps)-1]
+}
+
+// trunkReferences is metadata for a textNode indicating that the sub-tree
+// represents the value for either pointer in a pair of references.
+type trunkReferences struct{ pp [2]value.Pointer }
+
+// trunkReference is metadata for a textNode indicating that the sub-tree
+// represents the value for the given pointer reference.
+type trunkReference struct{ p value.Pointer }
+
+// leafReference is metadata for a textNode indicating that the value is
+// truncated as it refers to another part of the tree (i.e., a trunk).
+type leafReference struct{ p value.Pointer }
+
+func wrapTrunkReferences(pp [2]value.Pointer, s textNode) textNode {
+	switch {
+	case pp[0].IsNil():
+		return &textWrap{Value: s, Metadata: trunkReference{pp[1]}}
+	case pp[1].IsNil():
+		return &textWrap{Value: s, Metadata: trunkReference{pp[0]}}
+	case pp[0] == pp[1]:
+		return &textWrap{Value: s, Metadata: trunkReference{pp[0]}}
+	default:
+		return &textWrap{Value: s, Metadata: trunkReferences{pp}}
+	}
+}
+func wrapTrunkReference(p value.Pointer, printAddress bool, s textNode) textNode {
+	var prefix string
+	if printAddress {
+		prefix = formatPointer(p, true)
+	}
+	return &textWrap{Prefix: prefix, Value: s, Metadata: trunkReference{p}}
+}
+func makeLeafReference(p value.Pointer, printAddress bool) textNode {
+	out := &textWrap{Prefix: "(", Value: textEllipsis, Suffix: ")"}
+	var prefix string
+	if printAddress {
+		prefix = formatPointer(p, true)
+	}
+	return &textWrap{Prefix: prefix, Value: out, Metadata: leafReference{p}}
+}
+
+// resolveReferences walks the textNode tree searching for any leaf reference
+// metadata and resolves each against the corresponding trunk references.
+// Since pointer addresses in memory are not particularly readable to the user,
+// it replaces each pointer value with an arbitrary and unique reference ID.
+func resolveReferences(s textNode) {
+	var walkNodes func(textNode, func(textNode))
+	walkNodes = func(s textNode, f func(textNode)) {
+		f(s)
+		switch s := s.(type) {
+		case *textWrap:
+			walkNodes(s.Value, f)
+		case textList:
+			for _, r := range s {
+				walkNodes(r.Value, f)
+			}
+		}
+	}
+
+	// Collect all trunks and leaves with reference metadata.
+	var trunks, leaves []*textWrap
+	walkNodes(s, func(s textNode) {
+		if s, ok := s.(*textWrap); ok {
+			switch s.Metadata.(type) {
+			case leafReference:
+				leaves = append(leaves, s)
+			case trunkReference, trunkReferences:
+				trunks = append(trunks, s)
+			}
+		}
+	})
+
+	// No leaf references to resolve.
+	if len(leaves) == 0 {
+		return
+	}
+
+	// Collect the set of all leaf references to resolve.
+	leafPtrs := make(map[value.Pointer]bool)
+	for _, leaf := range leaves {
+		leafPtrs[leaf.Metadata.(leafReference).p] = true
+	}
+
+	// Collect the set of trunk pointers that are always paired together.
+	// This allows us to assign a single ID to both pointers for brevity.
+	// If a pointer in a pair ever occurs by itself or as a different pair,
+	// then the pair is broken.
+	pairedTrunkPtrs := make(map[value.Pointer]value.Pointer)
+	unpair := func(p value.Pointer) {
+		if !pairedTrunkPtrs[p].IsNil() {
+			pairedTrunkPtrs[pairedTrunkPtrs[p]] = value.Pointer{} // invalidate other half
+		}
+		pairedTrunkPtrs[p] = value.Pointer{} // invalidate this half
+	}
+	for _, trunk := range trunks {
+		switch p := trunk.Metadata.(type) {
+		case trunkReference:
+			unpair(p.p) // standalone pointer cannot be part of a pair
+		case trunkReferences:
+			p0, ok0 := pairedTrunkPtrs[p.pp[0]]
+			p1, ok1 := pairedTrunkPtrs[p.pp[1]]
+			switch {
+			case !ok0 && !ok1:
+				// Register the newly seen pair.
+				pairedTrunkPtrs[p.pp[0]] = p.pp[1]
+				pairedTrunkPtrs[p.pp[1]] = p.pp[0]
+			case ok0 && ok1 && p0 == p.pp[1] && p1 == p.pp[0]:
+				// Exact pair already seen; do nothing.
+			default:
+				// Pair conflicts with some other pair; break all pairs.
+				unpair(p.pp[0])
+				unpair(p.pp[1])
+			}
+		}
+	}
+
+	// Correlate each pointer referenced by leaves to a unique identifier,
+	// and print the IDs for each trunk that matches those pointers.
+	var nextID uint
+	ptrIDs := make(map[value.Pointer]uint)
+	newID := func() uint {
+		id := nextID
+		nextID++
+		return id
+	}
+	for _, trunk := range trunks {
+		switch p := trunk.Metadata.(type) {
+		case trunkReference:
+			if print := leafPtrs[p.p]; print {
+				id, ok := ptrIDs[p.p]
+				if !ok {
+					id = newID()
+					ptrIDs[p.p] = id
+				}
+				trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id))
+			}
+		case trunkReferences:
+			print0 := leafPtrs[p.pp[0]]
+			print1 := leafPtrs[p.pp[1]]
+			if print0 || print1 {
+				id0, ok0 := ptrIDs[p.pp[0]]
+				id1, ok1 := ptrIDs[p.pp[1]]
+				isPair := pairedTrunkPtrs[p.pp[0]] == p.pp[1] && pairedTrunkPtrs[p.pp[1]] == p.pp[0]
+				if isPair {
+					var id uint
+					assert(ok0 == ok1) // must be seen together or not at all
+					if ok0 {
+						assert(id0 == id1) // must have the same ID
+						id = id0
+					} else {
+						id = newID()
+						ptrIDs[p.pp[0]] = id
+						ptrIDs[p.pp[1]] = id
+					}
+					trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id))
+				} else {
+					if print0 && !ok0 {
+						id0 = newID()
+						ptrIDs[p.pp[0]] = id0
+					}
+					if print1 && !ok1 {
+						id1 = newID()
+						ptrIDs[p.pp[1]] = id1
+					}
+					switch {
+					case print0 && print1:
+						trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id0)+","+formatReference(id1))
+					case print0:
+						trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id0))
+					case print1:
+						trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id1))
+					}
+				}
+			}
+		}
+	}
+
+	// Update all leaf references with the unique identifier.
+	for _, leaf := range leaves {
+		if id, ok := ptrIDs[leaf.Metadata.(leafReference).p]; ok {
+			leaf.Prefix = updateReferencePrefix(leaf.Prefix, formatReference(id))
+		}
+	}
+}
+
+func formatReference(id uint) string {
+	return fmt.Sprintf("ref#%d", id)
+}
+
+func updateReferencePrefix(prefix, ref string) string {
+	if prefix == "" {
+		return pointerDelimPrefix + ref + pointerDelimSuffix
+	}
+	suffix := strings.TrimPrefix(prefix, pointerDelimPrefix)
+	return pointerDelimPrefix + ref + ": " + suffix
+}

vendor/github.com/google/go-cmp/cmp/report_slices.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -23,11 +24,25 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 		return false // Must be formatting in diff mode
 	case v.NumDiff == 0:
 		return false // No differences detected
-	case v.NumIgnored+v.NumCompared+v.NumTransformed > 0:
-		// TODO: Handle the case where someone uses bytes.Equal on a large slice.
-		return false // Some custom option was used to determined equality
 	case !v.ValueX.IsValid() || !v.ValueY.IsValid():
 		return false // Both values must be valid
+	case v.Type.Kind() == reflect.Slice && (v.ValueX.Len() == 0 || v.ValueY.Len() == 0):
+		return false // Both slice values have to be non-empty
+	case v.NumIgnored > 0:
+		return false // Some ignore option was used
+	case v.NumTransformed > 0:
+		return false // Some transform option was used
+	case v.NumCompared > 1:
+		return false // More than one comparison was used
+	case v.NumCompared == 1 && v.Type.Name() != "":
+		// The need for cmp to check applicability of options on every element
+		// in a slice is a significant performance detriment for large []byte.
+		// The workaround is to specify Comparer(bytes.Equal),
+		// which enables cmp to compare []byte more efficiently.
+		// If they differ, we still want to provide batched diffing.
+		// The logic disallows named types since they tend to have their own
+		// String method, with nicer formatting than what this provides.
+		return false
 	}
 
 	switch t := v.Type; t.Kind() {
@@ -82,7 +97,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	}
 	if isText || isBinary {
 		var numLines, lastLineIdx, maxLineLen int
-		isBinary = false
+		isBinary = !utf8.ValidString(sx) || !utf8.ValidString(sy)
 		for i, r := range sx + sy {
 			if !(unicode.IsPrint(r) || unicode.IsSpace(r)) || r == utf8.RuneError {
 				isBinary = true
@@ -97,7 +112,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 			}
 		}
 		isText = !isBinary
-		isLinedText = isText && numLines >= 4 && maxLineLen <= 256
+		isLinedText = isText && numLines >= 4 && maxLineLen <= 1024
 	}
 
 	// Format the string into printable records.
@@ -117,6 +132,83 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 			},
 		)
 		delim = "\n"
+
+		// If possible, use a custom triple-quote (""") syntax for printing
+		// differences in a string literal. This format is more readable,
+		// but has edge-cases where differences are visually indistinguishable.
+		// This format is avoided under the following conditions:
+		//	• A line starts with `"""`
+		//	• A line starts with "..."
+		//	• A line contains non-printable characters
+		//	• Adjacent different lines differ only by whitespace
+		//
+		// For example:
+		//		"""
+		//		... // 3 identical lines
+		//		foo
+		//		bar
+		//	-	baz
+		//	+	BAZ
+		//		"""
+		isTripleQuoted := true
+		prevRemoveLines := map[string]bool{}
+		prevInsertLines := map[string]bool{}
+		var list2 textList
+		list2 = append(list2, textRecord{Value: textLine(`"""`), ElideComma: true})
+		for _, r := range list {
+			if !r.Value.Equal(textEllipsis) {
+				line, _ := strconv.Unquote(string(r.Value.(textLine)))
+				line = strings.TrimPrefix(strings.TrimSuffix(line, "\r"), "\r") // trim leading/trailing carriage returns for legacy Windows endline support
+				normLine := strings.Map(func(r rune) rune {
+					if unicode.IsSpace(r) {
+						return -1 // drop whitespace to avoid visually indistinguishable output
+					}
+					return r
+				}, line)
+				isPrintable := func(r rune) bool {
+					return unicode.IsPrint(r) || r == '\t' // specially treat tab as printable
+				}
+				isTripleQuoted = !strings.HasPrefix(line, `"""`) && !strings.HasPrefix(line, "...") && strings.TrimFunc(line, isPrintable) == ""
+				switch r.Diff {
+				case diffRemoved:
+					isTripleQuoted = isTripleQuoted && !prevInsertLines[normLine]
+					prevRemoveLines[normLine] = true
+				case diffInserted:
+					isTripleQuoted = isTripleQuoted && !prevRemoveLines[normLine]
+					prevInsertLines[normLine] = true
+				}
+				if !isTripleQuoted {
+					break
+				}
+				r.Value = textLine(line)
+				r.ElideComma = true
+			}
+			if !(r.Diff == diffRemoved || r.Diff == diffInserted) { // start a new non-adjacent difference group
+				prevRemoveLines = map[string]bool{}
+				prevInsertLines = map[string]bool{}
+			}
+			list2 = append(list2, r)
+		}
+		if r := list2[len(list2)-1]; r.Diff == diffIdentical && len(r.Value.(textLine)) == 0 {
+			list2 = list2[:len(list2)-1] // elide single empty line at the end
+		}
+		list2 = append(list2, textRecord{Value: textLine(`"""`), ElideComma: true})
+		if isTripleQuoted {
+			var out textNode = &textWrap{Prefix: "(", Value: list2, Suffix: ")"}
+			switch t.Kind() {
+			case reflect.String:
+				if t != reflect.TypeOf(string("")) {
+					out = opts.FormatType(t, out)
+				}
+			case reflect.Slice:
+				// Always emit type for slices since the triple-quote syntax
+				// looks like a string (not a slice).
+				opts = opts.WithTypeMode(emitType)
+				out = opts.FormatType(t, out)
+			}
+			return out
+		}
+
 	// If the text appears to be single-lined text,
 	// then perform differencing in approximately fixed-sized chunks.
 	// The output is printed as quoted strings.
@@ -129,6 +221,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 			},
 		)
 		delim = ""
+
 	// If the text appears to be binary data,
 	// then perform differencing in approximately fixed-sized chunks.
 	// The output is inspired by hexdump.
@@ -145,6 +238,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 				return textRecord{Diff: d, Value: textLine(s), Comment: comment}
 			},
 		)
+
 	// For all other slices of primitive types,
 	// then perform differencing in approximately fixed-sized chunks.
 	// The size of each chunk depends on the width of the element kind.
@@ -172,7 +266,9 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 					switch t.Elem().Kind() {
 					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 						ss = append(ss, fmt.Sprint(v.Index(i).Int()))
-					case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+					case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+						ss = append(ss, fmt.Sprint(v.Index(i).Uint()))
+					case reflect.Uint8, reflect.Uintptr:
 						ss = append(ss, formatHex(v.Index(i).Uint()))
 					case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
 						ss = append(ss, fmt.Sprint(v.Index(i).Interface()))
@@ -185,7 +281,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	}
 
 	// Wrap the output with appropriate type information.
-	var out textNode = textWrap{"{", list, "}"}
+	var out textNode = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
 	if !isText {
 		// The "{...}" byte-sequence literal is not valid Go syntax for strings.
 		// Emit the type for extra clarity (e.g. "string{...}").
@@ -196,12 +292,12 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	}
 	switch t.Kind() {
 	case reflect.String:
-		out = textWrap{"strings.Join(", out, fmt.Sprintf(", %q)", delim)}
+		out = &textWrap{Prefix: "strings.Join(", Value: out, Suffix: fmt.Sprintf(", %q)", delim)}
 		if t != reflect.TypeOf(string("")) {
 			out = opts.FormatType(t, out)
 		}
 	case reflect.Slice:
-		out = textWrap{"bytes.Join(", out, fmt.Sprintf(", %q)", delim)}
+		out = &textWrap{Prefix: "bytes.Join(", Value: out, Suffix: fmt.Sprintf(", %q)", delim)}
 		if t != reflect.TypeOf([]byte(nil)) {
 			out = opts.FormatType(t, out)
 		}
@@ -242,9 +338,22 @@ func (opts formatOptions) formatDiffSlice(
 		return n0 - v.Len()
 	}
 
+	var numDiffs int
+	maxLen := -1
+	if opts.LimitVerbosity {
+		maxLen = (1 << opts.verbosity()) << 2 // 4, 8, 16, 32, 64, etc...
+		opts.VerbosityLevel--
+	}
+
 	groups := coalesceAdjacentEdits(name, es)
 	groups = coalesceInterveningIdentical(groups, chunkSize/4)
+	maxGroup := diffStats{Name: name}
 	for i, ds := range groups {
+		if maxLen >= 0 && numDiffs >= maxLen {
+			maxGroup = maxGroup.Append(ds)
+			continue
+		}
+
 		// Print equal.
 		if ds.NumDiff() == 0 {
 			// Compute the number of leading and trailing equal bytes to print.
@@ -273,12 +382,18 @@ func (opts formatOptions) formatDiffSlice(
 		}
 
 		// Print unequal.
+		len0 := len(list)
 		nx := appendChunks(vx.Slice(0, ds.NumIdentical+ds.NumRemoved+ds.NumModified), diffRemoved)
 		vx = vx.Slice(nx, vx.Len())
 		ny := appendChunks(vy.Slice(0, ds.NumIdentical+ds.NumInserted+ds.NumModified), diffInserted)
 		vy = vy.Slice(ny, vy.Len())
+		numDiffs += len(list) - len0
+	}
+	if maxGroup.IsZero() {
+		assert(vx.Len() == 0 && vy.Len() == 0)
+	} else {
+		list.AppendEllipsis(maxGroup)
 	}
-	assert(vx.Len() == 0 && vy.Len() == 0)
 	return list
 }
 

vendor/github.com/google/go-cmp/cmp/report_text.go

@@ -10,12 +10,15 @@ import (
 	"math/rand"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/google/go-cmp/cmp/internal/flags"
 )
 
 var randBool = rand.New(rand.NewSource(time.Now().Unix())).Intn(2) == 0
 
+const maxColumnLength = 80
+
 type indentMode int
 
 func (n indentMode) appendIndent(b []byte, d diffMode) []byte {
@@ -91,21 +94,22 @@ type textNode interface {
 // textWrap is a wrapper that concatenates a prefix and/or a suffix
 // to the underlying node.
 type textWrap struct {
-	Prefix string   // e.g., "bytes.Buffer{"
-	Value  textNode // textWrap | textList | textLine
-	Suffix string   // e.g., "}"
+	Prefix   string      // e.g., "bytes.Buffer{"
+	Value    textNode    // textWrap | textList | textLine
+	Suffix   string      // e.g., "}"
+	Metadata interface{} // arbitrary metadata; has no effect on formatting
 }
 
-func (s textWrap) Len() int {
+func (s *textWrap) Len() int {
 	return len(s.Prefix) + s.Value.Len() + len(s.Suffix)
 }
-func (s1 textWrap) Equal(s2 textNode) bool {
-	if s2, ok := s2.(textWrap); ok {
+func (s1 *textWrap) Equal(s2 textNode) bool {
+	if s2, ok := s2.(*textWrap); ok {
 		return s1.Prefix == s2.Prefix && s1.Value.Equal(s2.Value) && s1.Suffix == s2.Suffix
 	}
 	return false
 }
-func (s textWrap) String() string {
+func (s *textWrap) String() string {
 	var d diffMode
 	var n indentMode
 	_, s2 := s.formatCompactTo(nil, d)
@@ -114,7 +118,7 @@ func (s textWrap) String() string {
 	b = append(b, '\n')              // Trailing newline
 	return string(b)
 }
-func (s textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
+func (s *textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
 	n0 := len(b) // Original buffer length
 	b = append(b, s.Prefix...)
 	b, s.Value = s.Value.formatCompactTo(b, d)
@@ -124,7 +128,7 @@ func (s textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
 	}
 	return b, s
 }
-func (s textWrap) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
+func (s *textWrap) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 	b = append(b, s.Prefix...)
 	b = s.Value.formatExpandedTo(b, d, n)
 	b = append(b, s.Suffix...)
@@ -136,22 +140,23 @@ func (s textWrap) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 // of the textList.formatCompactTo method.
 type textList []textRecord
 type textRecord struct {
-	Diff    diffMode     // e.g., 0 or '-' or '+'
-	Key     string       // e.g., "MyField"
-	Value   textNode     // textWrap | textLine
-	Comment fmt.Stringer // e.g., "6 identical fields"
+	Diff       diffMode     // e.g., 0 or '-' or '+'
+	Key        string       // e.g., "MyField"
+	Value      textNode     // textWrap | textLine
+	ElideComma bool         // avoid trailing comma
+	Comment    fmt.Stringer // e.g., "6 identical fields"
 }
 
 // AppendEllipsis appends a new ellipsis node to the list if none already
 // exists at the end. If cs is non-zero it coalesces the statistics with the
 // previous diffStats.
 func (s *textList) AppendEllipsis(ds diffStats) {
-	hasStats := ds != diffStats{}
+	hasStats := !ds.IsZero()
 	if len(*s) == 0 || !(*s)[len(*s)-1].Value.Equal(textEllipsis) {
 		if hasStats {
-			*s = append(*s, textRecord{Value: textEllipsis, Comment: ds})
+			*s = append(*s, textRecord{Value: textEllipsis, ElideComma: true, Comment: ds})
 		} else {
-			*s = append(*s, textRecord{Value: textEllipsis})
+			*s = append(*s, textRecord{Value: textEllipsis, ElideComma: true})
 		}
 		return
 	}
@@ -191,7 +196,7 @@ func (s1 textList) Equal(s2 textNode) bool {
 }
 
 func (s textList) String() string {
-	return textWrap{"{", s, "}"}.String()
+	return (&textWrap{Prefix: "{", Value: s, Suffix: "}"}).String()
 }
 
 func (s textList) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
@@ -221,7 +226,7 @@ func (s textList) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
 	}
 	// Force multi-lined output when printing a removed/inserted node that
 	// is sufficiently long.
-	if (d == diffInserted || d == diffRemoved) && len(b[n0:]) > 80 {
+	if (d == diffInserted || d == diffRemoved) && len(b[n0:]) > maxColumnLength {
 		multiLine = true
 	}
 	if !multiLine {
@@ -236,16 +241,50 @@ func (s textList) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 			_, isLine := r.Value.(textLine)
 			return r.Key == "" || !isLine
 		},
-		func(r textRecord) int { return len(r.Key) },
+		func(r textRecord) int { return utf8.RuneCountInString(r.Key) },
 	)
 	alignValueLens := s.alignLens(
 		func(r textRecord) bool {
 			_, isLine := r.Value.(textLine)
 			return !isLine || r.Value.Equal(textEllipsis) || r.Comment == nil
 		},
-		func(r textRecord) int { return len(r.Value.(textLine)) },
+		func(r textRecord) int { return utf8.RuneCount(r.Value.(textLine)) },
 	)
 
+	// Format lists of simple lists in a batched form.
+	// If the list is sequence of only textLine values,
+	// then batch multiple values on a single line.
+	var isSimple bool
+	for _, r := range s {
+		_, isLine := r.Value.(textLine)
+		isSimple = r.Diff == 0 && r.Key == "" && isLine && r.Comment == nil
+		if !isSimple {
+			break
+		}
+	}
+	if isSimple {
+		n++
+		var batch []byte
+		emitBatch := func() {
+			if len(batch) > 0 {
+				b = n.appendIndent(append(b, '\n'), d)
+				b = append(b, bytes.TrimRight(batch, " ")...)
+				batch = batch[:0]
+			}
+		}
+		for _, r := range s {
+			line := r.Value.(textLine)
+			if len(batch)+len(line)+len(", ") > maxColumnLength {
+				emitBatch()
+			}
+			batch = append(batch, line...)
+			batch = append(batch, ", "...)
+		}
+		emitBatch()
+		n--
+		return n.appendIndent(append(b, '\n'), d)
+	}
+
 	// Format the list as a multi-lined output.
 	n++
 	for i, r := range s {
@@ -256,7 +295,7 @@ func (s textList) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 		b = alignKeyLens[i].appendChar(b, ' ')
 
 		b = r.Value.formatExpandedTo(b, d|r.Diff, n)
-		if !r.Value.Equal(textEllipsis) {
+		if !r.ElideComma {
 			b = append(b, ',')
 		}
 		b = alignValueLens[i].appendChar(b, ' ')
@@ -332,6 +371,11 @@ type diffStats struct {
 	NumModified  int
 }
 
+func (s diffStats) IsZero() bool {
+	s.Name = ""
+	return s == diffStats{}
+}
+
 func (s diffStats) NumDiff() int {
 	return s.NumRemoved + s.NumInserted + s.NumModified
 }

vendor/github.com/pierrec/lz4/.gitignore

+# Created by https://www.gitignore.io/api/macos
+
+### macOS ###
+*.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# End of https://www.gitignore.io/api/macos
+
+cmd/*/*exe
+.idea
\ No newline at end of file

vendor/github.com/pierrec/lz4/.travis.yml

+language: go
+
+env:
+  - GO111MODULE=off
+
+go:
+  - 1.9.x
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+  - master
+
+matrix:
+ fast_finish: true
+ allow_failures:
+   - go: master
+
+sudo: false
+
+script: 
+ - go test -v -cpu=2
+ - go test -v -cpu=2 -race
+ - go test -v -cpu=2 -tags noasm
+ - go test -v -cpu=2 -race -tags noasm

vendor/github.com/pierrec/lz4/LICENSE

+Copyright (c) 2015, Pierre Curto
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of xxHash nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

vendor/github.com/pierrec/lz4/README.md

+# lz4 : LZ4 compression in pure Go
+
+[![GoDoc](https://godoc.org/github.com/pierrec/lz4?status.svg)](https://godoc.org/github.com/pierrec/lz4)
+[![Build Status](https://travis-ci.org/pierrec/lz4.svg?branch=master)](https://travis-ci.org/pierrec/lz4)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pierrec/lz4)](https://goreportcard.com/report/github.com/pierrec/lz4)
+[![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/pierrec/lz4.svg?style=social)](https://github.com/pierrec/lz4/tags)
+
+## Overview
+
+This package provides a streaming interface to [LZ4 data streams](http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html) as well as low level compress and uncompress functions for LZ4 data blocks.
+The implementation is based on the reference C [one](https://github.com/lz4/lz4).
+
+## Install
+
+Assuming you have the go toolchain installed:
+
+```
+go get github.com/pierrec/lz4
+```
+
+There is a command line interface tool to compress and decompress LZ4 files.
+
+```
+go install github.com/pierrec/lz4/cmd/lz4c
+```
+
+Usage
+
+```
+Usage of lz4c:
+  -version
+        print the program version
+
+Subcommands:
+Compress the given files or from stdin to stdout.
+compress [arguments] [<file name> ...]
+  -bc
+        enable block checksum
+  -l int
+        compression level (0=fastest)
+  -sc
+        disable stream checksum
+  -size string
+        block max size [64K,256K,1M,4M] (default "4M")
+
+Uncompress the given files or from stdin to stdout.
+uncompress [arguments] [<file name> ...]
+
+```
+
+
+## Example
+
+```
+// Compress and uncompress an input string.
+s := "hello world"
+r := strings.NewReader(s)
+
+// The pipe will uncompress the data from the writer.
+pr, pw := io.Pipe()
+zw := lz4.NewWriter(pw)
+zr := lz4.NewReader(pr)
+
+go func() {
+	// Compress the input string.
+	_, _ = io.Copy(zw, r)
+	_ = zw.Close() // Make sure the writer is closed
+	_ = pw.Close() // Terminate the pipe
+}()
+
+_, _ = io.Copy(os.Stdout, zr)
+
+// Output:
+// hello world
+```
+
+## Contributing
+
+Contributions are very welcome for bug fixing, performance improvements...!
+
+- Open an issue with a proper description
+- Send a pull request with appropriate test case(s)
+
+## Contributors
+
+Thanks to all [contributors](https://github.com/pierrec/lz4/graphs/contributors)  so far!
+
+Special thanks to [@Zariel](https://github.com/Zariel) for his asm implementation of the decoder.
+
+Special thanks to [@klauspost](https://github.com/klauspost) for his work on optimizing the code.

vendor/github.com/pierrec/lz4/debug.go

+// +build lz4debug
+
+package lz4
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+const debugFlag = true
+
+func debug(args ...interface{}) {
+	_, file, line, _ := runtime.Caller(1)
+	file = filepath.Base(file)
+
+	f := fmt.Sprintf("LZ4: %s:%d %s", file, line, args[0])
+	if f[len(f)-1] != '\n' {
+		f += "\n"
+	}
+	fmt.Fprintf(os.Stderr, f, args[1:]...)
+}

vendor/github.com/pierrec/lz4/debug_stub.go

+// +build !lz4debug
+
+package lz4
+
+const debugFlag = false
+
+func debug(args ...interface{}) {}

vendor/github.com/pierrec/lz4/decode_amd64.go

+// +build !appengine
+// +build gc
+// +build !noasm
+
+package lz4
+
+//go:noescape
+func decodeBlock(dst, src []byte) int

vendor/github.com/pierrec/lz4/decode_amd64.s

+// +build !appengine
+// +build gc
+// +build !noasm
+
+#include "textflag.h"
+
+// AX scratch
+// BX scratch
+// CX scratch
+// DX token
+//
+// DI &dst
+// SI &src
+// R8 &dst + len(dst)
+// R9 &src + len(src)
+// R11 &dst
+// R12 short output end
+// R13 short input end
+// func decodeBlock(dst, src []byte) int
+// using 50 bytes of stack currently
+TEXT ·decodeBlock(SB), NOSPLIT, $64-56
+	MOVQ dst_base+0(FP), DI
+	MOVQ DI, R11
+	MOVQ dst_len+8(FP), R8
+	ADDQ DI, R8
+
+	MOVQ src_base+24(FP), SI
+	MOVQ src_len+32(FP), R9
+	ADDQ SI, R9
+
+	// shortcut ends
+	// short output end
+	MOVQ R8, R12
+	SUBQ $32, R12
+	// short input end
+	MOVQ R9, R13
+	SUBQ $16, R13
+
+loop:
+	// for si < len(src)
+	CMPQ SI, R9
+	JGE end
+
+	// token := uint32(src[si])
+	MOVBQZX (SI), DX
+	INCQ SI
+
+	// lit_len = token >> 4
+	// if lit_len > 0
+	// CX = lit_len
+	MOVQ DX, CX
+	SHRQ $4, CX
+
+	// if lit_len != 0xF
+	CMPQ CX, $0xF
+	JEQ lit_len_loop_pre
+	CMPQ DI, R12
+	JGE lit_len_loop_pre
+	CMPQ SI, R13
+	JGE lit_len_loop_pre
+
+	// copy shortcut
+
+	// A two-stage shortcut for the most common case:
+	// 1) If the literal length is 0..14, and there is enough space,
+	// enter the shortcut and copy 16 bytes on behalf of the literals
+	// (in the fast mode, only 8 bytes can be safely copied this way).
+	// 2) Further if the match length is 4..18, copy 18 bytes in a similar
+	// manner; but we ensure that there's enough space in the output for
+	// those 18 bytes earlier, upon entering the shortcut (in other words,
+	// there is a combined check for both stages).
+
+	// copy literal
+	MOVOU (SI), X0
+	MOVOU X0, (DI)
+	ADDQ CX, DI
+	ADDQ CX, SI
+
+	MOVQ DX, CX
+	ANDQ $0xF, CX
+
+	// The second stage: prepare for match copying, decode full info.
+	// If it doesn't work out, the info won't be wasted.
+	// offset := uint16(data[:2])
+	MOVWQZX (SI), DX
+	ADDQ $2, SI
+
+	MOVQ DI, AX
+	SUBQ DX, AX
+	CMPQ AX, DI
+	JGT err_short_buf
+
+	// if we can't do the second stage then jump straight to read the
+	// match length, we already have the offset.
+	CMPQ CX, $0xF
+	JEQ match_len_loop_pre
+	CMPQ DX, $8
+	JLT match_len_loop_pre
+	CMPQ AX, R11
+	JLT err_short_buf
+
+	// memcpy(op + 0, match + 0, 8);
+	MOVQ (AX), BX
+	MOVQ BX, (DI)
+	// memcpy(op + 8, match + 8, 8);
+	MOVQ 8(AX), BX
+	MOVQ BX, 8(DI)
+	// memcpy(op +16, match +16, 2);
+	MOVW 16(AX), BX
+	MOVW BX, 16(DI)
+
+	ADDQ $4, DI // minmatch
+	ADDQ CX, DI
+
+	// shortcut complete, load next token
+	JMP loop
+
+lit_len_loop_pre:
+	// if lit_len > 0
+	CMPQ CX, $0
+	JEQ offset
+	CMPQ CX, $0xF
+	JNE copy_literal
+
+lit_len_loop:
+	// for src[si] == 0xFF
+	CMPB (SI), $0xFF
+	JNE lit_len_finalise
+
+	// bounds check src[si+1]
+	MOVQ SI, AX
+	ADDQ $1, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	// lit_len += 0xFF
+	ADDQ $0xFF, CX
+	INCQ SI
+	JMP lit_len_loop
+
+lit_len_finalise:
+	// lit_len += int(src[si])
+	// si++
+	MOVBQZX (SI), AX
+	ADDQ AX, CX
+	INCQ SI
+
+copy_literal:
+	// bounds check src and dst
+	MOVQ SI, AX
+	ADDQ CX, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	MOVQ DI, AX
+	ADDQ CX, AX
+	CMPQ AX, R8
+	JGT err_short_buf
+
+	// whats a good cut off to call memmove?
+	CMPQ CX, $16
+	JGT memmove_lit
+
+	// if len(dst[di:]) < 16
+	MOVQ R8, AX
+	SUBQ DI, AX
+	CMPQ AX, $16
+	JLT memmove_lit
+
+	// if len(src[si:]) < 16
+	MOVQ R9, AX
+	SUBQ SI, AX
+	CMPQ AX, $16
+	JLT memmove_lit
+
+	MOVOU (SI), X0
+	MOVOU X0, (DI)
+
+	JMP finish_lit_copy
+
+memmove_lit:
+	// memmove(to, from, len)
+	MOVQ DI, 0(SP)
+	MOVQ SI, 8(SP)
+	MOVQ CX, 16(SP)
+	// spill
+	MOVQ DI, 24(SP)
+	MOVQ SI, 32(SP)
+	MOVQ CX, 40(SP) // need len to inc SI, DI after
+	MOVB DX, 48(SP)
+	CALL runtime·memmove(SB)
+
+	// restore registers
+	MOVQ 24(SP), DI
+	MOVQ 32(SP), SI
+	MOVQ 40(SP), CX
+	MOVB 48(SP), DX
+
+	// recalc initial values
+	MOVQ dst_base+0(FP), R8
+	MOVQ R8, R11
+	ADDQ dst_len+8(FP), R8
+	MOVQ src_base+24(FP), R9
+	ADDQ src_len+32(FP), R9
+	MOVQ R8, R12
+	SUBQ $32, R12
+	MOVQ R9, R13
+	SUBQ $16, R13
+
+finish_lit_copy:
+	ADDQ CX, SI
+	ADDQ CX, DI
+
+	CMPQ SI, R9
+	JGE end
+
+offset:
+	// CX := mLen
+	// free up DX to use for offset
+	MOVQ DX, CX
+
+	MOVQ SI, AX
+	ADDQ $2, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	// offset
+	// DX := int(src[si]) | int(src[si+1])<<8
+	MOVWQZX (SI), DX
+	ADDQ $2, SI
+
+	// 0 offset is invalid
+	CMPQ DX, $0
+	JEQ err_corrupt
+
+	ANDB $0xF, CX
+
+match_len_loop_pre:
+	// if mlen != 0xF
+	CMPB CX, $0xF
+	JNE copy_match
+
+match_len_loop:
+	// for src[si] == 0xFF
+	// lit_len += 0xFF
+	CMPB (SI), $0xFF
+	JNE match_len_finalise
+
+	// bounds check src[si+1]
+	MOVQ SI, AX
+	ADDQ $1, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	ADDQ $0xFF, CX
+	INCQ SI
+	JMP match_len_loop
+
+match_len_finalise:
+	// lit_len += int(src[si])
+	// si++
+	MOVBQZX (SI), AX
+	ADDQ AX, CX
+	INCQ SI
+
+copy_match:
+	// mLen += minMatch
+	ADDQ $4, CX
+
+	// check we have match_len bytes left in dst
+	// di+match_len < len(dst)
+	MOVQ DI, AX
+	ADDQ CX, AX
+	CMPQ AX, R8
+	JGT err_short_buf
+
+	// DX = offset
+	// CX = match_len
+	// BX = &dst + (di - offset)
+	MOVQ DI, BX
+	SUBQ DX, BX
+
+	// check BX is within dst
+	// if BX < &dst
+	CMPQ BX, R11
+	JLT err_short_buf
+
+	// if offset + match_len < di
+	MOVQ BX, AX
+	ADDQ CX, AX
+	CMPQ DI, AX
+	JGT copy_interior_match
+
+	// AX := len(dst[:di])
+	// MOVQ DI, AX
+	// SUBQ R11, AX
+
+	// copy 16 bytes at a time
+	// if di-offset < 16 copy 16-(di-offset) bytes to di
+	// then do the remaining
+
+copy_match_loop:
+	// for match_len >= 0
+	// dst[di] = dst[i]
+	// di++
+	// i++
+	MOVB (BX), AX
+	MOVB AX, (DI)
+	INCQ DI
+	INCQ BX
+	DECQ CX
+
+	CMPQ CX, $0
+	JGT copy_match_loop
+
+	JMP loop
+
+copy_interior_match:
+	CMPQ CX, $16
+	JGT memmove_match
+
+	// if len(dst[di:]) < 16
+	MOVQ R8, AX
+	SUBQ DI, AX
+	CMPQ AX, $16
+	JLT memmove_match
+
+	MOVOU (BX), X0
+	MOVOU X0, (DI)
+
+	ADDQ CX, DI
+	JMP loop
+
+memmove_match:
+	// memmove(to, from, len)
+	MOVQ DI, 0(SP)
+	MOVQ BX, 8(SP)
+	MOVQ CX, 16(SP)
+	// spill
+	MOVQ DI, 24(SP)
+	MOVQ SI, 32(SP)
+	MOVQ CX, 40(SP) // need len to inc SI, DI after
+	CALL runtime·memmove(SB)
+
+	// restore registers
+	MOVQ 24(SP), DI
+	MOVQ 32(SP), SI
+	MOVQ 40(SP), CX
+
+	// recalc initial values
+	MOVQ dst_base+0(FP), R8
+	MOVQ R8, R11 // TODO: make these sensible numbers
+	ADDQ dst_len+8(FP), R8
+	MOVQ src_base+24(FP), R9
+	ADDQ src_len+32(FP), R9
+	MOVQ R8, R12
+	SUBQ $32, R12
+	MOVQ R9, R13
+	SUBQ $16, R13
+
+	ADDQ CX, DI
+	JMP loop
+
+err_corrupt:
+	MOVQ $-1, ret+48(FP)
+	RET
+
+err_short_buf:
+	MOVQ $-2, ret+48(FP)
+	RET
+
+end:
+	SUBQ R11, DI
+	MOVQ DI, ret+48(FP)
+	RET

vendor/github.com/pierrec/lz4/decode_other.go

+// +build !amd64 appengine !gc noasm
+
+package lz4
+
+func decodeBlock(dst, src []byte) (ret int) {
+	const hasError = -2
+	defer func() {
+		if recover() != nil {
+			ret = hasError
+		}
+	}()
+
+	var si, di int
+	for {
+		// Literals and match lengths (token).
+		b := int(src[si])
+		si++
+
+		// Literals.
+		if lLen := b >> 4; lLen > 0 {
+			switch {
+			case lLen < 0xF && si+16 < len(src):
+				// Shortcut 1
+				// if we have enough room in src and dst, and the literals length
+				// is small enough (0..14) then copy all 16 bytes, even if not all
+				// are part of the literals.
+				copy(dst[di:], src[si:si+16])
+				si += lLen
+				di += lLen
+				if mLen := b & 0xF; mLen < 0xF {
+					// Shortcut 2
+					// if the match length (4..18) fits within the literals, then copy
+					// all 18 bytes, even if not all are part of the literals.
+					mLen += 4
+					if offset := int(src[si]) | int(src[si+1])<<8; mLen <= offset {
+						i := di - offset
+						end := i + 18
+						if end > len(dst) {
+							// The remaining buffer may not hold 18 bytes.
+							// See https://github.com/pierrec/lz4/issues/51.
+							end = len(dst)
+						}
+						copy(dst[di:], dst[i:end])
+						si += 2
+						di += mLen
+						continue
+					}
+				}
+			case lLen == 0xF:
+				for src[si] == 0xFF {
+					lLen += 0xFF
+					si++
+				}
+				lLen += int(src[si])
+				si++
+				fallthrough
+			default:
+				copy(dst[di:di+lLen], src[si:si+lLen])
+				si += lLen
+				di += lLen
+			}
+		}
+		if si >= len(src) {
+			return di
+		}
+
+		offset := int(src[si]) | int(src[si+1])<<8
+		if offset == 0 {
+			return hasError
+		}
+		si += 2
+
+		// Match.
+		mLen := b & 0xF
+		if mLen == 0xF {
+			for src[si] == 0xFF {
+				mLen += 0xFF
+				si++
+			}
+			mLen += int(src[si])
+			si++
+		}
+		mLen += minMatch
+
+		// Copy the match.
+		expanded := dst[di-offset:]
+		if mLen > offset {
+			// Efficiently copy the match dst[di-offset:di] into the dst slice.
+			bytesToCopy := offset * (mLen / offset)
+			for n := offset; n <= bytesToCopy+offset; n *= 2 {
+				copy(expanded[n:], expanded[:n])
+			}
+			di += bytesToCopy
+			mLen -= bytesToCopy
+		}
+		di += copy(dst[di:di+mLen], expanded[:mLen])
+	}
+}

vendor/github.com/pierrec/lz4/errors.go

+package lz4
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	rdebug "runtime/debug"
+)
+
+var (
+	// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
+	// block is corrupted or the destination buffer is not large enough for the uncompressed data.
+	ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
+	// ErrInvalid is returned when reading an invalid LZ4 archive.
+	ErrInvalid = errors.New("lz4: bad magic number")
+	// ErrBlockDependency is returned when attempting to decompress an archive created with block dependency.
+	ErrBlockDependency = errors.New("lz4: block dependency not supported")
+	// ErrUnsupportedSeek is returned when attempting to Seek any way but forward from the current position.
+	ErrUnsupportedSeek = errors.New("lz4: can only seek forward from io.SeekCurrent")
+)
+
+func recoverBlock(e *error) {
+	if r := recover(); r != nil && *e == nil {
+		if debugFlag {
+			fmt.Fprintln(os.Stderr, r)
+			rdebug.PrintStack()
+		}
+		*e = ErrInvalidSourceShortBuffer
+	}
+}

vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go

+// Package xxh32 implements the very fast XXH hashing algorithm (32 bits version).
+// (https://github.com/Cyan4973/XXH/)
+package xxh32
+
+import (
+	"encoding/binary"
+)
+
+const (
+	prime1 uint32 = 2654435761
+	prime2 uint32 = 2246822519
+	prime3 uint32 = 3266489917
+	prime4 uint32 = 668265263
+	prime5 uint32 = 374761393
+
+	primeMask   = 0xFFFFFFFF
+	prime1plus2 = uint32((uint64(prime1) + uint64(prime2)) & primeMask) // 606290984
+	prime1minus = uint32((-int64(prime1)) & primeMask)                  // 1640531535
+)
+
+// XXHZero represents an xxhash32 object with seed 0.
+type XXHZero struct {
+	v1       uint32
+	v2       uint32
+	v3       uint32
+	v4       uint32
+	totalLen uint64
+	buf      [16]byte
+	bufused  int
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (xxh XXHZero) Sum(b []byte) []byte {
+	h32 := xxh.Sum32()
+	return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24))
+}
+
+// Reset resets the Hash to its initial state.
+func (xxh *XXHZero) Reset() {
+	xxh.v1 = prime1plus2
+	xxh.v2 = prime2
+	xxh.v3 = 0
+	xxh.v4 = prime1minus
+	xxh.totalLen = 0
+	xxh.bufused = 0
+}
+
+// Size returns the number of bytes returned by Sum().
+func (xxh *XXHZero) Size() int {
+	return 4
+}
+
+// BlockSize gives the minimum number of bytes accepted by Write().
+func (xxh *XXHZero) BlockSize() int {
+	return 1
+}
+
+// Write adds input bytes to the Hash.
+// It never returns an error.
+func (xxh *XXHZero) Write(input []byte) (int, error) {
+	if xxh.totalLen == 0 {
+		xxh.Reset()
+	}
+	n := len(input)
+	m := xxh.bufused
+
+	xxh.totalLen += uint64(n)
+
+	r := len(xxh.buf) - m
+	if n < r {
+		copy(xxh.buf[m:], input)
+		xxh.bufused += len(input)
+		return n, nil
+	}
+
+	p := 0
+	// Causes compiler to work directly from registers instead of stack:
+	v1, v2, v3, v4 := xxh.v1, xxh.v2, xxh.v3, xxh.v4
+	if m > 0 {
+		// some data left from previous update
+		copy(xxh.buf[xxh.bufused:], input[:r])
+		xxh.bufused += len(input) - r
+
+		// fast rotl(13)
+		buf := xxh.buf[:16] // BCE hint.
+		v1 = rol13(v1+binary.LittleEndian.Uint32(buf[:])*prime2) * prime1
+		v2 = rol13(v2+binary.LittleEndian.Uint32(buf[4:])*prime2) * prime1
+		v3 = rol13(v3+binary.LittleEndian.Uint32(buf[8:])*prime2) * prime1
+		v4 = rol13(v4+binary.LittleEndian.Uint32(buf[12:])*prime2) * prime1
+		p = r
+		xxh.bufused = 0
+	}
+
+	for n := n - 16; p <= n; p += 16 {
+		sub := input[p:][:16] //BCE hint for compiler
+		v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1
+		v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1
+		v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1
+		v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1
+	}
+	xxh.v1, xxh.v2, xxh.v3, xxh.v4 = v1, v2, v3, v4
+
+	copy(xxh.buf[xxh.bufused:], input[p:])
+	xxh.bufused += len(input) - p
+
+	return n, nil
+}
+
+// Sum32 returns the 32 bits Hash value.
+func (xxh *XXHZero) Sum32() uint32 {
+	h32 := uint32(xxh.totalLen)
+	if h32 >= 16 {
+		h32 += rol1(xxh.v1) + rol7(xxh.v2) + rol12(xxh.v3) + rol18(xxh.v4)
+	} else {
+		h32 += prime5
+	}
+
+	p := 0
+	n := xxh.bufused
+	buf := xxh.buf
+	for n := n - 4; p <= n; p += 4 {
+		h32 += binary.LittleEndian.Uint32(buf[p:p+4]) * prime3
+		h32 = rol17(h32) * prime4
+	}
+	for ; p < n; p++ {
+		h32 += uint32(buf[p]) * prime5
+		h32 = rol11(h32) * prime1
+	}
+
+	h32 ^= h32 >> 15
+	h32 *= prime2
+	h32 ^= h32 >> 13
+	h32 *= prime3
+	h32 ^= h32 >> 16
+
+	return h32
+}
+
+// ChecksumZero returns the 32bits Hash value.
+func ChecksumZero(input []byte) uint32 {
+	n := len(input)
+	h32 := uint32(n)
+
+	if n < 16 {
+		h32 += prime5
+	} else {
+		v1 := prime1plus2
+		v2 := prime2
+		v3 := uint32(0)
+		v4 := prime1minus
+		p := 0
+		for n := n - 16; p <= n; p += 16 {
+			sub := input[p:][:16] //BCE hint for compiler
+			v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1
+			v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1
+			v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1
+			v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1
+		}
+		input = input[p:]
+		n -= p
+		h32 += rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
+	}
+
+	p := 0
+	for n := n - 4; p <= n; p += 4 {
+		h32 += binary.LittleEndian.Uint32(input[p:p+4]) * prime3
+		h32 = rol17(h32) * prime4
+	}
+	for p < n {
+		h32 += uint32(input[p]) * prime5
+		h32 = rol11(h32) * prime1
+		p++
+	}
+
+	h32 ^= h32 >> 15
+	h32 *= prime2
+	h32 ^= h32 >> 13
+	h32 *= prime3
+	h32 ^= h32 >> 16
+
+	return h32
+}
+
+// Uint32Zero hashes x with seed 0.
+func Uint32Zero(x uint32) uint32 {
+	h := prime5 + 4 + x*prime3
+	h = rol17(h) * prime4
+	h ^= h >> 15
+	h *= prime2
+	h ^= h >> 13
+	h *= prime3
+	h ^= h >> 16
+	return h
+}
+
+func rol1(u uint32) uint32 {
+	return u<<1 | u>>31
+}
+
+func rol7(u uint32) uint32 {
+	return u<<7 | u>>25
+}
+
+func rol11(u uint32) uint32 {
+	return u<<11 | u>>21
+}
+
+func rol12(u uint32) uint32 {
+	return u<<12 | u>>20
+}
+
+func rol13(u uint32) uint32 {
+	return u<<13 | u>>19
+}
+
+func rol17(u uint32) uint32 {
+	return u<<17 | u>>15
+}
+
+func rol18(u uint32) uint32 {
+	return u<<18 | u>>14
+}

vendor/github.com/pierrec/lz4/lz4.go

+// Package lz4 implements reading and writing lz4 compressed data (a frame),
+// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
+//
+// Although the block level compression and decompression functions are exposed and are fully compatible
+// with the lz4 block format definition, they are low level and should not be used directly.
+// For a complete description of an lz4 compressed block, see:
+// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
+//
+// See https://github.com/Cyan4973/lz4 for the reference C implementation.
+//
+package lz4
+
+import "math/bits"
+
+import "sync"
+
+const (
+	// Extension is the LZ4 frame file name extension
+	Extension = ".lz4"
+	// Version is the LZ4 frame format version
+	Version = 1
+
+	frameMagic     uint32 = 0x184D2204
+	frameSkipMagic uint32 = 0x184D2A50
+
+	// The following constants are used to setup the compression algorithm.
+	minMatch            = 4  // the minimum size of the match sequence size (4 bytes)
+	winSizeLog          = 16 // LZ4 64Kb window size limit
+	winSize             = 1 << winSizeLog
+	winMask             = winSize - 1 // 64Kb window of previous data for dependent blocks
+	compressedBlockFlag = 1 << 31
+	compressedBlockMask = compressedBlockFlag - 1
+
+	// hashLog determines the size of the hash table used to quickly find a previous match position.
+	// Its value influences the compression speed and memory usage, the lower the faster,
+	// but at the expense of the compression ratio.
+	// 16 seems to be the best compromise for fast compression.
+	hashLog = 16
+	htSize  = 1 << hashLog
+
+	mfLimit = 10 + minMatch // The last match cannot start within the last 14 bytes.
+)
+
+// map the block max size id with its value in bytes: 64Kb, 256Kb, 1Mb and 4Mb.
+const (
+	blockSize64K = 1 << (16 + 2*iota)
+	blockSize256K
+	blockSize1M
+	blockSize4M
+)
+
+var (
+	// Keep a pool of buffers for each valid block sizes.
+	bsMapValue = [...]*sync.Pool{
+		newBufferPool(2 * blockSize64K),
+		newBufferPool(2 * blockSize256K),
+		newBufferPool(2 * blockSize1M),
+		newBufferPool(2 * blockSize4M),
+	}
+)
+
+// newBufferPool returns a pool for buffers of the given size.
+func newBufferPool(size int) *sync.Pool {
+	return &sync.Pool{
+		New: func() interface{} {
+			return make([]byte, size)
+		},
+	}
+}
+
+// getBuffer returns a buffer to its pool.
+func getBuffer(size int) []byte {
+	idx := blockSizeValueToIndex(size) - 4
+	return bsMapValue[idx].Get().([]byte)
+}
+
+// putBuffer returns a buffer to its pool.
+func putBuffer(size int, buf []byte) {
+	if cap(buf) > 0 {
+		idx := blockSizeValueToIndex(size) - 4
+		bsMapValue[idx].Put(buf[:cap(buf)])
+	}
+}
+func blockSizeIndexToValue(i byte) int {
+	return 1 << (16 + 2*uint(i))
+}
+func isValidBlockSize(size int) bool {
+	const blockSizeMask = blockSize64K | blockSize256K | blockSize1M | blockSize4M
+
+	return size&blockSizeMask > 0 && bits.OnesCount(uint(size)) == 1
+}
+func blockSizeValueToIndex(size int) byte {
+	return 4 + byte(bits.TrailingZeros(uint(size)>>16)/2)
+}
+
+// Header describes the various flags that can be set on a Writer or obtained from a Reader.
+// The default values match those of the LZ4 frame format definition
+// (http://fastcompression.blogspot.com/2013/04/lz4-streaming-format-final.html).
+//
+// NB. in a Reader, in case of concatenated frames, the Header values may change between Read() calls.
+// It is the caller's responsibility to check them if necessary.
+type Header struct {
+	BlockChecksum    bool   // Compressed blocks checksum flag.
+	NoChecksum       bool   // Frame checksum flag.
+	BlockMaxSize     int    // Size of the uncompressed data block (one of [64KB, 256KB, 1MB, 4MB]). Default=4MB.
+	Size             uint64 // Frame total size. It is _not_ computed by the Writer.
+	CompressionLevel int    // Compression level (higher is better, use 0 for fastest compression).
+	done             bool   // Header processed flag (Read or Write and checked).
+}
+
+func (h *Header) Reset() {
+	h.done = false
+}

vendor/github.com/pierrec/lz4/lz4_go1.10.go

+//+build go1.10
+
+package lz4
+
+import (
+	"fmt"
+	"strings"
+)
+
+func (h Header) String() string {
+	var s strings.Builder
+
+	s.WriteString(fmt.Sprintf("%T{", h))
+	if h.BlockChecksum {
+		s.WriteString("BlockChecksum: true ")
+	}
+	if h.NoChecksum {
+		s.WriteString("NoChecksum: true ")
+	}
+	if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 {
+		s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs))
+	}
+	if l := h.CompressionLevel; l != 0 {
+		s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l))
+	}
+	s.WriteByte('}')
+
+	return s.String()
+}

vendor/github.com/pierrec/lz4/lz4_notgo1.10.go

+//+build !go1.10
+
+package lz4
+
+import (
+	"bytes"
+	"fmt"
+)
+
+func (h Header) String() string {
+	var s bytes.Buffer
+
+	s.WriteString(fmt.Sprintf("%T{", h))
+	if h.BlockChecksum {
+		s.WriteString("BlockChecksum: true ")
+	}
+	if h.NoChecksum {
+		s.WriteString("NoChecksum: true ")
+	}
+	if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 {
+		s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs))
+	}
+	if l := h.CompressionLevel; l != 0 {
+		s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l))
+	}
+	s.WriteByte('}')
+
+	return s.String()
+}

vendor/github.com/pierrec/lz4/reader.go

+package lz4
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/pierrec/lz4/internal/xxh32"
+)
+
+// Reader implements the LZ4 frame decoder.
+// The Header is set after the first call to Read().
+// The Header may change between Read() calls in case of concatenated frames.
+type Reader struct {
+	Header
+	// Handler called when a block has been successfully read.
+	// It provides the number of bytes read.
+	OnBlockDone func(size int)
+
+	buf      [8]byte       // Scrap buffer.
+	pos      int64         // Current position in src.
+	src      io.Reader     // Source.
+	zdata    []byte        // Compressed data.
+	data     []byte        // Uncompressed data.
+	idx      int           // Index of unread bytes into data.
+	checksum xxh32.XXHZero // Frame hash.
+	skip     int64         // Bytes to skip before next read.
+	dpos     int64         // Position in dest
+}
+
+// NewReader returns a new LZ4 frame decoder.
+// No access to the underlying io.Reader is performed.
+func NewReader(src io.Reader) *Reader {
+	r := &Reader{src: src}
+	return r
+}
+
+// readHeader checks the frame magic number and parses the frame descriptoz.
+// Skippable frames are supported even as a first frame although the LZ4
+// specifications recommends skippable frames not to be used as first frames.
+func (z *Reader) readHeader(first bool) error {
+	defer z.checksum.Reset()
+
+	buf := z.buf[:]
+	for {
+		magic, err := z.readUint32()
+		if err != nil {
+			z.pos += 4
+			if !first && err == io.ErrUnexpectedEOF {
+				return io.EOF
+			}
+			return err
+		}
+		if magic == frameMagic {
+			break
+		}
+		if magic>>8 != frameSkipMagic>>8 {
+			return ErrInvalid
+		}
+		skipSize, err := z.readUint32()
+		if err != nil {
+			return err
+		}
+		z.pos += 4
+		m, err := io.CopyN(ioutil.Discard, z.src, int64(skipSize))
+		if err != nil {
+			return err
+		}
+		z.pos += m
+	}
+
+	// Header.
+	if _, err := io.ReadFull(z.src, buf[:2]); err != nil {
+		return err
+	}
+	z.pos += 8
+
+	b := buf[0]
+	if v := b >> 6; v != Version {
+		return fmt.Errorf("lz4: invalid version: got %d; expected %d", v, Version)
+	}
+	if b>>5&1 == 0 {
+		return ErrBlockDependency
+	}
+	z.BlockChecksum = b>>4&1 > 0
+	frameSize := b>>3&1 > 0
+	z.NoChecksum = b>>2&1 == 0
+
+	bmsID := buf[1] >> 4 & 0x7
+	if bmsID < 4 || bmsID > 7 {
+		return fmt.Errorf("lz4: invalid block max size ID: %d", bmsID)
+	}
+	bSize := blockSizeIndexToValue(bmsID - 4)
+	z.BlockMaxSize = bSize
+
+	// Allocate the compressed/uncompressed buffers.
+	// The compressed buffer cannot exceed the uncompressed one.
+	if n := 2 * bSize; cap(z.zdata) < n {
+		z.zdata = make([]byte, n, n)
+	}
+	if debugFlag {
+		debug("header block max size id=%d size=%d", bmsID, bSize)
+	}
+	z.zdata = z.zdata[:bSize]
+	z.data = z.zdata[:cap(z.zdata)][bSize:]
+	z.idx = len(z.data)
+
+	_, _ = z.checksum.Write(buf[0:2])
+
+	if frameSize {
+		buf := buf[:8]
+		if _, err := io.ReadFull(z.src, buf); err != nil {
+			return err
+		}
+		z.Size = binary.LittleEndian.Uint64(buf)
+		z.pos += 8
+		_, _ = z.checksum.Write(buf)
+	}
+
+	// Header checksum.
+	if _, err := io.ReadFull(z.src, buf[:1]); err != nil {
+		return err
+	}
+	z.pos++
+	if h := byte(z.checksum.Sum32() >> 8 & 0xFF); h != buf[0] {
+		return fmt.Errorf("lz4: invalid header checksum: got %x; expected %x", buf[0], h)
+	}
+
+	z.Header.done = true
+	if debugFlag {
+		debug("header read: %v", z.Header)
+	}
+
+	return nil
+}
+
+// Read decompresses data from the underlying source into the supplied buffer.
+//
+// Since there can be multiple streams concatenated, Header values may
+// change between calls to Read(). If that is the case, no data is actually read from
+// the underlying io.Reader, to allow for potential input buffer resizing.
+func (z *Reader) Read(buf []byte) (int, error) {
+	if debugFlag {
+		debug("Read buf len=%d", len(buf))
+	}
+	if !z.Header.done {
+		if err := z.readHeader(true); err != nil {
+			return 0, err
+		}
+		if debugFlag {
+			debug("header read OK compressed buffer %d / %d uncompressed buffer %d : %d index=%d",
+				len(z.zdata), cap(z.zdata), len(z.data), cap(z.data), z.idx)
+		}
+	}
+
+	if len(buf) == 0 {
+		return 0, nil
+	}
+
+	if z.idx == len(z.data) {
+		// No data ready for reading, process the next block.
+		if debugFlag {
+			debug("reading block from writer")
+		}
+		// Reset uncompressed buffer
+		z.data = z.zdata[:cap(z.zdata)][len(z.zdata):]
+
+		// Block length: 0 = end of frame, highest bit set: uncompressed.
+		bLen, err := z.readUint32()
+		if err != nil {
+			return 0, err
+		}
+		z.pos += 4
+
+		if bLen == 0 {
+			// End of frame reached.
+			if !z.NoChecksum {
+				// Validate the frame checksum.
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				if debugFlag {
+					debug("frame checksum got=%x / want=%x", z.checksum.Sum32(), checksum)
+				}
+				z.pos += 4
+				if h := z.checksum.Sum32(); checksum != h {
+					return 0, fmt.Errorf("lz4: invalid frame checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+			// Get ready for the next concatenated frame and keep the position.
+			pos := z.pos
+			z.Reset(z.src)
+			z.pos = pos
+
+			// Since multiple frames can be concatenated, check for more.
+			return 0, z.readHeader(false)
+		}
+
+		if debugFlag {
+			debug("raw block size %d", bLen)
+		}
+		if bLen&compressedBlockFlag > 0 {
+			// Uncompressed block.
+			bLen &= compressedBlockMask
+			if debugFlag {
+				debug("uncompressed block size %d", bLen)
+			}
+			if int(bLen) > cap(z.data) {
+				return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
+			}
+			z.data = z.data[:bLen]
+			if _, err := io.ReadFull(z.src, z.data); err != nil {
+				return 0, err
+			}
+			z.pos += int64(bLen)
+			if z.OnBlockDone != nil {
+				z.OnBlockDone(int(bLen))
+			}
+
+			if z.BlockChecksum {
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				z.pos += 4
+
+				if h := xxh32.ChecksumZero(z.data); h != checksum {
+					return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+		} else {
+			// Compressed block.
+			if debugFlag {
+				debug("compressed block size %d", bLen)
+			}
+			if int(bLen) > cap(z.data) {
+				return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
+			}
+			zdata := z.zdata[:bLen]
+			if _, err := io.ReadFull(z.src, zdata); err != nil {
+				return 0, err
+			}
+			z.pos += int64(bLen)
+
+			if z.BlockChecksum {
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				z.pos += 4
+
+				if h := xxh32.ChecksumZero(zdata); h != checksum {
+					return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+			n, err := UncompressBlock(zdata, z.data)
+			if err != nil {
+				return 0, err
+			}
+			z.data = z.data[:n]
+			if z.OnBlockDone != nil {
+				z.OnBlockDone(n)
+			}
+		}
+
+		if !z.NoChecksum {
+			_, _ = z.checksum.Write(z.data)
+			if debugFlag {
+				debug("current frame checksum %x", z.checksum.Sum32())
+			}
+		}
+		z.idx = 0
+	}
+
+	if z.skip > int64(len(z.data[z.idx:])) {
+		z.skip -= int64(len(z.data[z.idx:]))
+		z.dpos += int64(len(z.data[z.idx:]))
+		z.idx = len(z.data)
+		return 0, nil
+	}
+
+	z.idx += int(z.skip)
+	z.dpos += z.skip
+	z.skip = 0
+
+	n := copy(buf, z.data[z.idx:])
+	z.idx += n
+	z.dpos += int64(n)
+	if debugFlag {
+		debug("copied %d bytes to input", n)
+	}
+
+	return n, nil
+}
+
+// Seek implements io.Seeker, but supports seeking forward from the current
+// position only. Any other seek will return an error. Allows skipping output
+// bytes which aren't needed, which in some scenarios is faster than reading
+// and discarding them.
+// Note this may cause future calls to Read() to read 0 bytes if all of the
+// data they would have returned is skipped.
+func (z *Reader) Seek(offset int64, whence int) (int64, error) {
+	if offset < 0 || whence != io.SeekCurrent {
+		return z.dpos + z.skip, ErrUnsupportedSeek
+	}
+	z.skip += offset
+	return z.dpos + z.skip, nil
+}
+
+// Reset discards the Reader's state and makes it equivalent to the
+// result of its original state from NewReader, but reading from r instead.
+// This permits reusing a Reader rather than allocating a new one.
+func (z *Reader) Reset(r io.Reader) {
+	z.Header = Header{}
+	z.pos = 0
+	z.src = r
+	z.zdata = z.zdata[:0]
+	z.data = z.data[:0]
+	z.idx = 0
+	z.checksum.Reset()
+}
+
+// readUint32 reads an uint32 into the supplied buffer.
+// The idea is to make use of the already allocated buffers avoiding additional allocations.
+func (z *Reader) readUint32() (uint32, error) {
+	buf := z.buf[:4]
+	_, err := io.ReadFull(z.src, buf)
+	x := binary.LittleEndian.Uint32(buf)
+	return x, err
+}

vendor/modules.txt

@@ -532,7 +532,7 @@ github.com/google/cadvisor/utils/sysinfo
 github.com/google/cadvisor/version
 github.com/google/cadvisor/watcher
 github.com/google/cadvisor/zfs
-# github.com/google/go-cmp v0.3.1
+# github.com/google/go-cmp v0.5.2
 github.com/google/go-cmp/cmp
 github.com/google/go-cmp/cmp/internal/diff
 github.com/google/go-cmp/cmp/internal/flags
@@ -702,6 +702,9 @@ github.com/opencontainers/selinux/go-selinux/label
 github.com/opencontainers/selinux/pkg/pwalk
 # github.com/peterbourgon/diskv v2.0.1+incompatible
 github.com/peterbourgon/diskv
+# github.com/pierrec/lz4 v2.5.2+incompatible
+github.com/pierrec/lz4
+github.com/pierrec/lz4/internal/xxh32
 # github.com/pkg/errors v0.9.1
 github.com/pkg/errors
 # github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021