Commit 7d3447ce authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson

Bump wharfie to v0.5.1 and use shared decompression code

parent 05d43278
......@@ -21,6 +21,7 @@ replace (
github.com/docker/docker => github.com/docker/docker v20.10.2+incompatible
github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34
github.com/golang/protobuf => github.com/golang/protobuf v1.5.2
github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.0.5
github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f
github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.21.0-k3s1
github.com/matryer/moq => github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009
......@@ -81,7 +82,7 @@ require (
github.com/containerd/stargz-snapshotter v0.8.0
github.com/coreos/go-iptables v0.5.0
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
github.com/docker/docker v20.10.7+incompatible
github.com/docker/docker v20.10.10+incompatible
github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
github.com/flannel-io/flannel v0.15.1
github.com/go-bindata/go-bindata v3.1.2+incompatible
......@@ -93,7 +94,7 @@ require (
github.com/gorilla/websocket v1.4.2
github.com/k3s-io/helm-controller v0.11.7
github.com/k3s-io/kine v0.8.1
github.com/klauspost/compress v1.13.5
github.com/klauspost/compress v1.13.6
github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
github.com/lib/pq v1.10.2
github.com/mattn/go-sqlite3 v1.14.8
......@@ -105,12 +106,11 @@ require (
github.com/opencontainers/runc v1.0.2
github.com/opencontainers/selinux v1.8.2
github.com/otiai10/copy v1.6.0
github.com/pierrec/lz4 v2.6.0+incompatible
github.com/pkg/errors v0.9.1
github.com/rancher/dynamiclistener v0.3.1
github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08
github.com/rancher/remotedialer v0.2.0
github.com/rancher/wharfie v0.3.4
github.com/rancher/wharfie v0.5.1
github.com/rancher/wrangler v0.8.9
github.com/robfig/cron/v3 v3.0.1
github.com/rootless-containers/rootlesskit v0.14.5
......@@ -118,15 +118,15 @@ require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/tchap/go-patricia v2.3.0+incompatible // indirect
github.com/urfave/cli v1.22.2
github.com/urfave/cli v1.22.4
go.etcd.io/etcd/api/v3 v3.5.0
go.etcd.io/etcd/client/v3 v3.5.0
go.etcd.io/etcd/etcdutl/v3 v3.5.0
go.etcd.io/etcd/server/v3 v3.5.0
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
google.golang.org/grpc v1.40.0
golang.org/x/net v0.0.0-20211005215030-d2e5035098b3
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef
google.golang.org/grpc v1.41.0
gopkg.in/yaml.v2 v2.4.0
inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252
k8s.io/api v0.22.4
......
......@@ -57,7 +57,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
NodeConfig: cfg,
DisableCgroup: disableCgroup,
IsRunningInUserNS: isRunningInUserNS,
PrivateRegistryConfig: privRegistries.Registry(),
PrivateRegistryConfig: privRegistries.Registry,
ExtraRuntimes: findNvidiaContainerRuntimes(os.DirFS(string(os.PathSeparator))),
}
......
......@@ -2,8 +2,6 @@ package containerd
import (
"bufio"
"compress/bzip2"
"compress/gzip"
"context"
"fmt"
"io"
......@@ -21,14 +19,12 @@ import (
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/pkg/cri/constants"
"github.com/containerd/containerd/reference/docker"
"github.com/klauspost/compress/zstd"
"github.com/natefinch/lumberjack"
"github.com/pierrec/lz4"
"github.com/pkg/errors"
util2 "github.com/rancher/k3s/pkg/agent/util"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/untar"
"github.com/rancher/k3s/pkg/version"
"github.com/rancher/wharfie/pkg/tarfile"
"github.com/rancher/wrangler/pkg/merr"
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
......@@ -194,7 +190,7 @@ func preloadImages(ctx context.Context, cfg *config.Node) error {
logrus.Errorf("Error encountered while importing %s: %v", filePath, err)
continue
}
logrus.Debugf("Imported images from %s in %s", filePath, time.Since(start))
logrus.Infof("Imported images from %s in %s", filePath, time.Since(start))
}
return nil
}
......@@ -203,39 +199,26 @@ func preloadImages(ctx context.Context, cfg *config.Node) error {
// This is in its own function so that we can ensure that the various readers are properly closed, as some
// decompressing readers need to be explicitly closed and others do not.
func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Client, criConn *grpc.ClientConn, filePath string) error {
file, err := os.Open(filePath)
if util2.HasSuffixI(filePath, ".txt") {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
logrus.Infof("Pulling images from %s", filePath)
return prePullImages(ctx, criConn, file)
}
opener, err := tarfile.GetOpener(filePath)
if err != nil {
return err
}
defer file.Close()
var imageReader io.Reader
switch {
case util2.HasSuffixI(filePath, ".txt"):
return prePullImages(ctx, criConn, file)
case util2.HasSuffixI(filePath, ".tar"):
imageReader = file
case util2.HasSuffixI(filePath, ".tar.lz4"):
imageReader = lz4.NewReader(file)
case util2.HasSuffixI(filePath, ".tar.bz2", ".tbz"):
imageReader = bzip2.NewReader(file)
case util2.HasSuffixI(filePath, ".tar.gz", ".tgz"):
zr, err := gzip.NewReader(file)
if err != nil {
return err
}
defer zr.Close()
imageReader = zr
case util2.HasSuffixI(filePath, "tar.zst", ".tzst"):
zr, err := zstd.NewReader(file, zstd.WithDecoderMaxMemory(untar.MaxDecoderMemory))
if err != nil {
return err
}
defer zr.Close()
imageReader = zr
default:
return errors.New("unhandled file type")
imageReader, err := opener()
if err != nil {
return err
}
defer imageReader.Close()
logrus.Infof("Importing images from %s", filePath)
......
......@@ -15,20 +15,10 @@ import (
"time"
"github.com/klauspost/compress/zstd"
"github.com/rancher/wharfie/pkg/tarfile"
"github.com/sirupsen/logrus"
)
const (
// The zstd decoder will attempt to use up to 1GB memory for streaming operations by default,
// which is excessive and will OOM low-memory devices.
// NOTE: This must be at least as large as the window size used when compressing tarballs, or you
// will see a "window size exceeded" error when decompressing. The zstd CLI tool uses 4MB by
// default; the --long option defaults to 27 or 128M, which is still too much for a Pi3. 32MB
// (--long=25) has been tested to work acceptably while still compressing by an additional 3-6% on
// our datasets.
MaxDecoderMemory = 1 << 25
)
// TODO(bradfitz): this was copied from x/build/cmd/buildlet/buildlet.go
// but there were some buildlet-specific bits in there, so the code is
// forked for now. Unfork and add some opts arguments here, so the
......@@ -49,7 +39,7 @@ func untar(r io.Reader, dir string) (err error) {
logrus.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err)
}
}()
zr, err := zstd.NewReader(r, zstd.WithDecoderMaxMemory(MaxDecoderMemory))
zr, err := zstd.NewReader(r, zstd.WithDecoderMaxMemory(tarfile.MaxDecoderMemory))
if err != nil {
return fmt.Errorf("error extracting zstd-compressed body: %v", err)
}
......
......@@ -140,7 +140,7 @@ func testOnGCE() bool {
}()
go func() {
addrs, err := net.LookupHost("metadata.google.internal")
addrs, err := net.DefaultResolver.LookupHost(ctx, "metadata.google.internal")
if err != nil || len(addrs) == 0 {
resc <- false
return
......@@ -282,6 +282,7 @@ func NewClient(c *http.Client) *Client {
// getETag returns a value from the metadata service as well as the associated ETag.
// This func is otherwise equivalent to Get.
func (c *Client) getETag(suffix string) (value, etag string, err error) {
ctx := context.TODO()
// Using a fixed IP makes it very difficult to spoof the metadata service in
// a container, which is an important use-case for local testing of cloud
// deployments. To enable spoofing of the metadata service, the environment
......@@ -296,6 +297,7 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
// being stable anyway.
host = metadataIP
}
suffix = strings.TrimLeft(suffix, "/")
u := "http://" + host + "/computeMetadata/v1/" + suffix
req, err := http.NewRequest("GET", u, nil)
if err != nil {
......@@ -303,9 +305,25 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
}
req.Header.Set("Metadata-Flavor", "Google")
req.Header.Set("User-Agent", userAgent)
res, err := c.hc.Do(req)
if err != nil {
return "", "", err
var res *http.Response
var reqErr error
retryer := newRetryer()
for {
res, reqErr = c.hc.Do(req)
var code int
if res != nil {
code = res.StatusCode
}
if delay, shouldRetry := retryer.Retry(code, reqErr); shouldRetry {
if err := sleep(ctx, delay); err != nil {
return "", "", err
}
continue
}
break
}
if reqErr != nil {
return "", "", nil
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
......
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadata
import (
"context"
"io"
"math/rand"
"net/http"
"time"
)
const (
maxRetryAttempts = 5
)
var (
syscallRetryable = func(err error) bool { return false }
)
// defaultBackoff is basically equivalent to gax.Backoff without the need for
// the dependency.
type defaultBackoff struct {
max time.Duration
mul float64
cur time.Duration
}
func (b *defaultBackoff) Pause() time.Duration {
d := time.Duration(1 + rand.Int63n(int64(b.cur)))
b.cur = time.Duration(float64(b.cur) * b.mul)
if b.cur > b.max {
b.cur = b.max
}
return d
}
// sleep is the equivalent of gax.Sleep without the need for the dependency.
func sleep(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
select {
case <-ctx.Done():
t.Stop()
return ctx.Err()
case <-t.C:
return nil
}
}
func newRetryer() *metadataRetryer {
return &metadataRetryer{bo: &defaultBackoff{
cur: 100 * time.Millisecond,
max: 30 * time.Second,
mul: 2,
}}
}
type backoff interface {
Pause() time.Duration
}
type metadataRetryer struct {
bo backoff
attempts int
}
func (r *metadataRetryer) Retry(status int, err error) (time.Duration, bool) {
if status == http.StatusOK {
return 0, false
}
retryOk := shouldRetry(status, err)
if !retryOk {
return 0, false
}
if r.attempts == maxRetryAttempts {
return 0, false
}
r.attempts++
return r.bo.Pause(), true
}
func shouldRetry(status int, err error) bool {
if 500 <= status && status <= 599 {
return true
}
if err == io.ErrUnexpectedEOF {
return true
}
// Transient network errors should be retried.
if syscallRetryable(err) {
return true
}
if err, ok := err.(interface{ Temporary() bool }); ok {
if err.Temporary() {
return true
}
}
if err, ok := err.(interface{ Unwrap() error }); ok {
return shouldRetry(status, err.Unwrap())
}
return false
}
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
// +build linux
package metadata
import "syscall"
func init() {
// Initialize syscallRetryable to return true on transient socket-level
// errors. These errors are specific to Linux.
syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED }
}
# go-winio
# go-winio [![Build Status](https://github.com/microsoft/go-winio/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/go-winio/actions/workflows/ci.yml)
This repository contains utilities for efficiently performing Win32 IO operations in
Go. Currently, this is focused on accessing named pipes and other file handles, and
......
......@@ -28,8 +28,9 @@ const (
ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeSecurityPrivilege = "SeSecurityPrivilege"
)
const (
......
......@@ -23,7 +23,6 @@
package estargz
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
......@@ -42,6 +41,7 @@ import (
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/vbatts/tar-split/archive/tar"
)
// A Reader permits random access reads from a stargz file.
......@@ -95,10 +95,10 @@ func WithTelemetry(telemetry *Telemetry) OpenOption {
}
}
// A func which takes start time and records the diff
// MeasureLatencyHook is a func which takes start time and records the diff
type MeasureLatencyHook func(time.Time)
// A struct which defines telemetry hooks. By implementing these hooks you should be able to record
// Telemetry is a struct which defines telemetry hooks. By implementing these hooks you should be able to record
// the latency metrics of the respective steps of estargz open operation. To be used with estargz.OpenWithTelemetry(...)
type Telemetry struct {
GetFooterLatency MeasureLatencyHook // measure time to get stargz footer (in milliseconds)
......@@ -146,7 +146,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
fSize := d.FooterSize()
fOffset := positive(int64(len(footer)) - fSize)
maybeTocBytes := footer[:fOffset]
tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:])
_, tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:])
if err != nil {
allErr = append(allErr, err)
continue
......@@ -187,7 +187,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e
for _, d := range []Decompressor{new(GzipDecompressor), new(legacyGzipDecompressor)} {
fSize := d.FooterSize()
fOffset := positive(int64(len(footer)) - fSize)
tocOffset, _, err := d.ParseFooter(footer[fOffset:])
_, tocOffset, _, err := d.ParseFooter(footer[fOffset:])
if err == nil {
return tocOffset, fSize, err
}
......@@ -326,6 +326,10 @@ func (r *Reader) getOrCreateDir(d string) *TOCEntry {
return e
}
func (r *Reader) TOCDigest() digest.Digest {
return r.tocDigest
}
// VerifyTOC checks that the TOC JSON in the passed blob matches the
// passed digests and that the TOC JSON contains digests for all chunks
// contained in the blob. If the verification succceeds, this function
......@@ -335,7 +339,12 @@ func (r *Reader) VerifyTOC(tocDigest digest.Digest) (TOCEntryVerifier, error) {
if r.tocDigest != tocDigest {
return nil, fmt.Errorf("invalid TOC JSON %q; want %q", r.tocDigest, tocDigest)
}
return r.Verifiers()
}
// Verifiers returns TOCEntryVerifier of this chunk. Use VerifyTOC instead in most cases
// because this doesn't verify TOC.
func (r *Reader) Verifiers() (TOCEntryVerifier, error) {
chunkDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the chunk digest
regDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the reg file digest
var chunkDigestMapIncomplete bool
......@@ -591,6 +600,11 @@ type currentCompressionWriter struct{ w *Writer }
func (ccw currentCompressionWriter) Write(p []byte) (int, error) {
ccw.w.diffHash.Write(p)
if ccw.w.gz == nil {
if err := ccw.w.condOpenGz(); err != nil {
return 0, err
}
}
return ccw.w.gz.Write(p)
}
......@@ -601,6 +615,25 @@ func (w *Writer) chunkSize() int {
return w.ChunkSize
}
// Unpack decompresses the given estargz blob and returns a ReadCloser of the tar blob.
// TOC JSON and footer are removed.
func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) {
footerSize := c.FooterSize()
if sr.Size() < footerSize {
return nil, fmt.Errorf("blob is too small; %d < %d", sr.Size(), footerSize)
}
footerOffset := sr.Size() - footerSize
footer := make([]byte, footerSize)
if _, err := sr.ReadAt(footer, footerOffset); err != nil {
return nil, err
}
blobPayloadSize, _, _, err := c.ParseFooter(footer)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse footer")
}
return c.Reader(io.LimitReader(sr, blobPayloadSize))
}
// NewWriter returns a new stargz writer (gzip-based) writing to w.
//
// The writer must be closed to write its trailing table of contents.
......@@ -616,7 +649,7 @@ func NewWriterLevel(w io.Writer, compressionLevel int) *Writer {
return NewWriterWithCompressor(w, NewGzipCompressorWithLevel(compressionLevel))
}
// NewWriterLevel returns a new stargz writer writing to w.
// NewWriterWithCompressor returns a new stargz writer writing to w.
// The compression method is configurable.
//
// The writer must be closed to write its trailing table of contents.
......@@ -696,29 +729,71 @@ func (w *Writer) condOpenGz() (err error) {
// each of its contents to w.
//
// The input r can optionally be gzip compressed but the output will
// always be gzip compressed.
// always be compressed by the specified compressor.
func (w *Writer) AppendTar(r io.Reader) error {
return w.appendTar(r, false)
}
// AppendTarLossLess reads the tar or tar.gz file from r and appends
// each of its contents to w.
//
// The input r can optionally be gzip compressed but the output will
// always be compressed by the specified compressor.
//
// The difference of this func with AppendTar is that this writes
// the input tar stream into w without any modification (e.g. to header bytes).
//
// Note that if the input tar stream already contains TOC JSON, this returns
// error because w cannot overwrite the TOC JSON to the one generated by w without
// lossy modification. To avoid this error, if the input stream is known to be stargz/estargz,
// you shoud decompress it and remove TOC JSON in advance.
func (w *Writer) AppendTarLossLess(r io.Reader) error {
return w.appendTar(r, true)
}
func (w *Writer) appendTar(r io.Reader, lossless bool) error {
var src io.Reader
br := bufio.NewReader(r)
var tr *tar.Reader
if isGzip(br) {
// NewReader can't fail if isGzip returned true.
zr, _ := gzip.NewReader(br)
tr = tar.NewReader(zr)
src = zr
} else {
tr = tar.NewReader(br)
src = io.Reader(br)
}
dst := currentCompressionWriter{w}
var tw *tar.Writer
if !lossless {
tw = tar.NewWriter(dst) // use tar writer only when this isn't lossless mode.
}
tr := tar.NewReader(src)
if lossless {
tr.RawAccounting = true
}
for {
h, err := tr.Next()
if err == io.EOF {
if lossless {
if remain := tr.RawBytes(); len(remain) > 0 {
// Collect the remaining null bytes.
// https://github.com/vbatts/tar-split/blob/80a436fd6164c557b131f7c59ed69bd81af69761/concept/main.go#L49-L53
if _, err := dst.Write(remain); err != nil {
return err
}
}
}
break
}
if err != nil {
return fmt.Errorf("error reading from source tar: tar.Reader.Next: %v", err)
}
if h.Name == TOCTarName {
if cleanEntryName(h.Name) == TOCTarName {
// It is possible for a layer to be "stargzified" twice during the
// distribution lifecycle. So we reserve "TOCTarName" here to avoid
// duplicated entries in the resulting layer.
if lossless {
// We cannot handle this in lossless way.
return fmt.Errorf("existing TOC JSON is not allowed; decompress layer before append")
}
continue
}
......@@ -744,9 +819,14 @@ func (w *Writer) AppendTar(r io.Reader) error {
if err := w.condOpenGz(); err != nil {
return err
}
tw := tar.NewWriter(currentCompressionWriter{w})
if err := tw.WriteHeader(h); err != nil {
return err
if tw != nil {
if err := tw.WriteHeader(h); err != nil {
return err
}
} else {
if _, err := dst.Write(tr.RawBytes()); err != nil {
return err
}
}
switch h.Typeflag {
case tar.TypeLink:
......@@ -808,7 +888,13 @@ func (w *Writer) AppendTar(r io.Reader) error {
}
teeChunk := io.TeeReader(tee, chunkDigest.Hash())
if _, err := io.CopyN(tw, teeChunk, chunkSize); err != nil {
var out io.Writer
if tw != nil {
out = tw
} else {
out = dst
}
if _, err := io.CopyN(out, teeChunk, chunkSize); err != nil {
return fmt.Errorf("error copying %q: %v", h.Name, err)
}
ent.ChunkDigest = chunkDigest.Digest().String()
......@@ -825,11 +911,18 @@ func (w *Writer) AppendTar(r io.Reader) error {
if payloadDigest != nil {
regFileEntry.Digest = payloadDigest.Digest().String()
}
if err := tw.Flush(); err != nil {
return err
if tw != nil {
if err := tw.Flush(); err != nil {
return err
}
}
}
return nil
remainDest := ioutil.Discard
if lossless {
remainDest = dst // Preserve the remaining bytes in lossless mode
}
_, err := io.Copy(remainDest, src)
return err
}
// DiffID returns the SHA-256 of the uncompressed tar bytes.
......
......@@ -3,8 +3,9 @@ module github.com/containerd/stargz-snapshotter/estargz
go 1.16
require (
github.com/klauspost/compress v1.13.5
github.com/klauspost/compress v1.13.6
github.com/opencontainers/go-digest v1.0.0
github.com/pkg/errors v0.9.1
github.com/vbatts/tar-split v0.11.2
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
)
github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
......@@ -124,31 +124,31 @@ func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Dig
return parseTOCEStargz(r)
}
func (gz *GzipDecompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) {
func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
if len(p) != FooterSize {
return 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
}
zr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return 0, 0, err
return 0, 0, 0, err
}
defer zr.Close()
extra := zr.Header.Extra
si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
if si1 != 'S' || si2 != 'G' {
return 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
}
if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) {
return 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
}
if string(subfield[16:]) != "STARGZ" {
return 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
}
tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64)
if err != nil {
return 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
}
return tocOffset, 0, nil
return tocOffset, tocOffset, 0, nil
}
func (gz *GzipDecompressor) FooterSize() int64 {
......@@ -165,27 +165,27 @@ func (gz *legacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst dige
return parseTOCEStargz(r)
}
func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) {
func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
if len(p) != legacyFooterSize {
return 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p))
return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p))
}
zr, err := gzip.NewReader(bytes.NewReader(p))
if err != nil {
return 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader")
return 0, 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader")
}
defer zr.Close()
extra := zr.Header.Extra
if len(extra) != 16+len("STARGZ") {
return 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size")
return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size")
}
if string(extra[16:]) != "STARGZ" {
return 0, 0, fmt.Errorf("legacy: magic string STARGZ not found")
return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found")
}
tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64)
if err != nil {
return 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
}
return tocOffset, 0, nil
return tocOffset, tocOffset, 0, nil
}
func (gz *legacyGzipDecompressor) FooterSize() int64 {
......
......@@ -290,7 +290,7 @@ type Compressor interface {
WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error)
}
// Deompressor represents the helper mothods to be used for parsing eStargz.
// Decompressor represents the helper mothods to be used for parsing eStargz.
type Decompressor interface {
// Reader returns ReadCloser to be used for decompressing file payload.
Reader(r io.Reader) (io.ReadCloser, error)
......@@ -299,10 +299,12 @@ type Decompressor interface {
FooterSize() int64
// ParseFooter parses the footer and returns the offset and (compressed) size of TOC.
// payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between
// the top until the TOC JSON).
//
// Here, tocSize is optional. If tocSize <= 0, it's by default the size of the range
// from tocOffset until the beginning of the footer (blob size - tocOff - FooterSize).
ParseFooter(p []byte) (tocOffset, tocSize int64, err error)
ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error)
// ParseTOC parses TOC from the passed reader. The reader provides the partial contents
// of the underlying blob that has the range specified by ParseFooter method.
......
......@@ -32,10 +32,16 @@ import (
)
const (
ZstdChunkedManifestChecksumAnnotation = "io.containers.zstd-chunked.manifest-checksum"
ZstdChunkedManifestPositionAnnotation = "io.containers.zstd-chunked.manifest-position"
FooterSize = 40
manifestTypeCRFS = 1
// ManifestChecksumAnnotation is an annotation that contains the compressed TOC Digset
ManifestChecksumAnnotation = "io.containers.zstd-chunked.manifest-checksum"
// ManifestPositionAnnotation is an annotation that contains the offset to the TOC.
ManifestPositionAnnotation = "io.containers.zstd-chunked.manifest-position"
// FooterSize is the size of the footer
FooterSize = 40
manifestTypeCRFS = 1
)
var (
......@@ -68,13 +74,14 @@ func (zz *Decompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest
return toc, dgstr.Digest(), nil
}
func (zz *Decompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) {
func (zz *Decompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
offset := binary.LittleEndian.Uint64(p[0:8])
compressedLength := binary.LittleEndian.Uint64(p[8:16])
if !bytes.Equal(zstdChunkedFrameMagic, p[32:40]) {
return 0, 0, fmt.Errorf("invalid magic number")
return 0, 0, 0, fmt.Errorf("invalid magic number")
}
return int64(offset), int64(compressedLength), nil
// 8 is the size of the zstd skippable frame header + the frame size (see WriteTOCAndFooter)
return int64(offset - 8), int64(offset), int64(compressedLength), nil
}
func (zz *Decompressor) FooterSize() int64 {
......@@ -149,8 +156,8 @@ func (zc *Compressor) WriteTOCAndFooter(w io.Writer, off int64, toc *estargz.JTO
}
if zc.Metadata != nil {
zc.Metadata[ZstdChunkedManifestChecksumAnnotation] = digest.FromBytes(compressedTOC).String()
zc.Metadata[ZstdChunkedManifestPositionAnnotation] = fmt.Sprintf("%d:%d:%d:%d",
zc.Metadata[ManifestChecksumAnnotation] = digest.FromBytes(compressedTOC).String()
zc.Metadata[ManifestPositionAnnotation] = fmt.Sprintf("%d:%d:%d:%d",
tocOff, len(compressedTOC), len(tocJSON), manifestTypeCRFS)
}
......
......@@ -15,7 +15,7 @@ type roffRenderer struct {
extensions blackfriday.Extensions
listCounters []int
firstHeader bool
defineTerm bool
firstDD bool
listDepth int
}
......@@ -42,7 +42,8 @@ const (
quoteCloseTag = "\n.RE\n"
listTag = "\n.RS\n"
listCloseTag = "\n.RE\n"
arglistTag = "\n.TP\n"
dtTag = "\n.TP\n"
dd2Tag = "\n"
tableStart = "\n.TS\nallbox;\n"
tableEnd = ".TE\n"
tableCellStart = "T{\n"
......@@ -90,7 +91,7 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering
switch node.Type {
case blackfriday.Text:
r.handleText(w, node, entering)
escapeSpecialChars(w, node.Literal)
case blackfriday.Softbreak:
out(w, crTag)
case blackfriday.Hardbreak:
......@@ -150,40 +151,21 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering
out(w, codeCloseTag)
case blackfriday.Table:
r.handleTable(w, node, entering)
case blackfriday.TableCell:
r.handleTableCell(w, node, entering)
case blackfriday.TableHead:
case blackfriday.TableBody:
case blackfriday.TableRow:
// no action as cell entries do all the nroff formatting
return blackfriday.GoToNext
case blackfriday.TableCell:
r.handleTableCell(w, node, entering)
case blackfriday.HTMLSpan:
// ignore other HTML tags
default:
fmt.Fprintln(os.Stderr, "WARNING: go-md2man does not handle node type "+node.Type.String())
}
return walkAction
}
func (r *roffRenderer) handleText(w io.Writer, node *blackfriday.Node, entering bool) {
var (
start, end string
)
// handle special roff table cell text encapsulation
if node.Parent.Type == blackfriday.TableCell {
if len(node.Literal) > 30 {
start = tableCellStart
end = tableCellEnd
} else {
// end rows that aren't terminated by "tableCellEnd" with a cr if end of row
if node.Parent.Next == nil && !node.Parent.IsHeader {
end = crTag
}
}
}
out(w, start)
escapeSpecialChars(w, node.Literal)
out(w, end)
}
func (r *roffRenderer) handleHeading(w io.Writer, node *blackfriday.Node, entering bool) {
if entering {
switch node.Level {
......@@ -230,15 +212,20 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering
if node.ListFlags&blackfriday.ListTypeOrdered != 0 {
out(w, fmt.Sprintf(".IP \"%3d.\" 5\n", r.listCounters[len(r.listCounters)-1]))
r.listCounters[len(r.listCounters)-1]++
} else if node.ListFlags&blackfriday.ListTypeTerm != 0 {
// DT (definition term): line just before DD (see below).
out(w, dtTag)
r.firstDD = true
} else if node.ListFlags&blackfriday.ListTypeDefinition != 0 {
// state machine for handling terms and following definitions
// since blackfriday does not distinguish them properly, nor
// does it seperate them into separate lists as it should
if !r.defineTerm {
out(w, arglistTag)
r.defineTerm = true
// DD (definition description): line that starts with ": ".
//
// We have to distinguish between the first DD and the
// subsequent ones, as there should be no vertical
// whitespace between the DT and the first DD.
if r.firstDD {
r.firstDD = false
} else {
r.defineTerm = false
out(w, dd2Tag)
}
} else {
out(w, ".IP \\(bu 2\n")
......@@ -251,7 +238,7 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering
func (r *roffRenderer) handleTable(w io.Writer, node *blackfriday.Node, entering bool) {
if entering {
out(w, tableStart)
//call walker to count cells (and rows?) so format section can be produced
// call walker to count cells (and rows?) so format section can be produced
columns := countColumns(node)
out(w, strings.Repeat("l ", columns)+"\n")
out(w, strings.Repeat("l ", columns)+".\n")
......@@ -261,28 +248,41 @@ func (r *roffRenderer) handleTable(w io.Writer, node *blackfriday.Node, entering
}
func (r *roffRenderer) handleTableCell(w io.Writer, node *blackfriday.Node, entering bool) {
var (
start, end string
)
if node.IsHeader {
start = codespanTag
end = codespanCloseTag
}
if entering {
var start string
if node.Prev != nil && node.Prev.Type == blackfriday.TableCell {
out(w, "\t"+start)
} else {
out(w, start)
start = "\t"
}
if node.IsHeader {
start += codespanTag
} else if nodeLiteralSize(node) > 30 {
start += tableCellStart
}
out(w, start)
} else {
// need to carriage return if we are at the end of the header row
if node.IsHeader && node.Next == nil {
end = end + crTag
var end string
if node.IsHeader {
end = codespanCloseTag
} else if nodeLiteralSize(node) > 30 {
end = tableCellEnd
}
if node.Next == nil && end != tableCellEnd {
// Last cell: need to carriage return if we are at the end of the
// header row and content isn't wrapped in a "tablecell"
end += crTag
}
out(w, end)
}
}
func nodeLiteralSize(node *blackfriday.Node) int {
total := 0
for n := node.FirstChild; n != nil; n = n.FirstChild {
total += len(n.Literal)
}
return total
}
// because roff format requires knowing the column count before outputting any table
// data we need to walk a table tree and count the columns
func countColumns(node *blackfriday.Node) int {
......@@ -309,15 +309,6 @@ func out(w io.Writer, output string) {
io.WriteString(w, output) // nolint: errcheck
}
func needsBackslash(c byte) bool {
for _, r := range []byte("-_&\\~") {
if c == r {
return true
}
}
return false
}
func escapeSpecialChars(w io.Writer, text []byte) {
for i := 0; i < len(text); i++ {
// escape initial apostrophe or period
......@@ -328,7 +319,7 @@ func escapeSpecialChars(w io.Writer, text []byte) {
// directly copy normal characters
org := i
for i < len(text) && !needsBackslash(text[i]) {
for i < len(text) && text[i] != '\\' {
i++
}
if i > org {
......
package challenge
import (
"net/url"
"strings"
)
// FROM: https://golang.org/src/net/http/http.go
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
// FROM: http://golang.org/src/net/http/transport.go
var portMap = map[string]string{
"http": "80",
"https": "443",
}
// canonicalAddr returns url.Host but always with a ":port" suffix
// FROM: http://golang.org/src/net/http/transport.go
func canonicalAddr(url *url.URL) string {
addr := url.Host
if !hasPort(addr) {
return addr + ":" + portMap[url.Scheme]
}
return addr
}
package challenge
import (
"fmt"
"net/http"
"net/url"
"strings"
"sync"
)
// Challenge carries information from a WWW-Authenticate response header.
// See RFC 2617.
type Challenge struct {
// Scheme is the auth-scheme according to RFC 2617
Scheme string
// Parameters are the auth-params according to RFC 2617
Parameters map[string]string
}
// Manager manages the challenges for endpoints.
// The challenges are pulled out of HTTP responses. Only
// responses which expect challenges should be added to
// the manager, since a non-unauthorized request will be
// viewed as not requiring challenges.
type Manager interface {
// GetChallenges returns the challenges for the given
// endpoint URL.
GetChallenges(endpoint url.URL) ([]Challenge, error)
// AddResponse adds the response to the challenge
// manager. The challenges will be parsed out of
// the WWW-Authenicate headers and added to the
// URL which was produced the response. If the
// response was authorized, any challenges for the
// endpoint will be cleared.
AddResponse(resp *http.Response) error
}
// NewSimpleManager returns an instance of
// Manger which only maps endpoints to challenges
// based on the responses which have been added the
// manager. The simple manager will make no attempt to
// perform requests on the endpoints or cache the responses
// to a backend.
func NewSimpleManager() Manager {
return &simpleManager{
Challenges: make(map[string][]Challenge),
}
}
type simpleManager struct {
sync.RWMutex
Challenges map[string][]Challenge
}
func normalizeURL(endpoint *url.URL) {
endpoint.Host = strings.ToLower(endpoint.Host)
endpoint.Host = canonicalAddr(endpoint)
}
func (m *simpleManager) GetChallenges(endpoint url.URL) ([]Challenge, error) {
normalizeURL(&endpoint)
m.RLock()
defer m.RUnlock()
challenges := m.Challenges[endpoint.String()]
return challenges, nil
}
func (m *simpleManager) AddResponse(resp *http.Response) error {
challenges := ResponseChallenges(resp)
if resp.Request == nil {
return fmt.Errorf("missing request reference")
}
urlCopy := url.URL{
Path: resp.Request.URL.Path,
Host: resp.Request.URL.Host,
Scheme: resp.Request.URL.Scheme,
}
normalizeURL(&urlCopy)
m.Lock()
defer m.Unlock()
m.Challenges[urlCopy.String()] = challenges
return nil
}
// Octet types from RFC 2616.
type octetType byte
var octetTypes [256]octetType
const (
isToken octetType = 1 << iota
isSpace
)
func init() {
// OCTET = <any 8-bit sequence of data>
// CHAR = <any US-ASCII character (octets 0 - 127)>
// CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
// CR = <US-ASCII CR, carriage return (13)>
// LF = <US-ASCII LF, linefeed (10)>
// SP = <US-ASCII SP, space (32)>
// HT = <US-ASCII HT, horizontal-tab (9)>
// <"> = <US-ASCII double-quote mark (34)>
// CRLF = CR LF
// LWS = [CRLF] 1*( SP | HT )
// TEXT = <any OCTET except CTLs, but including LWS>
// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
// | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
// token = 1*<any CHAR except CTLs or separators>
// qdtext = <any TEXT except <">>
for c := 0; c < 256; c++ {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
t |= isSpace
}
if isChar && !isCtl && !isSeparator {
t |= isToken
}
octetTypes[c] = t
}
}
// ResponseChallenges returns a list of authorization challenges
// for the given http Response. Challenges are only checked if
// the response status code was a 401.
func ResponseChallenges(resp *http.Response) []Challenge {
if resp.StatusCode == http.StatusUnauthorized {
// Parse the WWW-Authenticate Header and store the challenges
// on this endpoint object.
return parseAuthHeader(resp.Header)
}
return nil
}
func parseAuthHeader(header http.Header) []Challenge {
challenges := []Challenge{}
for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] {
v, p := parseValueAndParams(h)
if v != "" {
challenges = append(challenges, Challenge{Scheme: v, Parameters: p})
}
}
return challenges
}
func parseValueAndParams(header string) (value string, params map[string]string) {
params = make(map[string]string)
value, s := expectToken(header)
if value == "" {
return
}
value = strings.ToLower(value)
s = "," + skipSpace(s)
for strings.HasPrefix(s, ",") {
var pkey string
pkey, s = expectToken(skipSpace(s[1:]))
if pkey == "" {
return
}
if !strings.HasPrefix(s, "=") {
return
}
var pvalue string
pvalue, s = expectTokenOrQuoted(s[1:])
if pvalue == "" {
return
}
pkey = strings.ToLower(pkey)
params[pkey] = pvalue
s = skipSpace(s)
}
return
}
func skipSpace(s string) (rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isSpace == 0 {
break
}
}
return s[i:]
}
func expectToken(s string) (token, rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isToken == 0 {
break
}
}
return s[:i], s[i:]
}
func expectTokenOrQuoted(s string) (value string, rest string) {
if !strings.HasPrefix(s, "\"") {
return expectToken(s)
}
s = s[1:]
for i := 0; i < len(s); i++ {
switch s[i] {
case '"':
return s[:i], s[i+1:]
case '\\':
p := make([]byte, len(s)-1)
j := copy(p, s[:i])
escape := true
for i = i + 1; i < len(s); i++ {
b := s[i]
switch {
case escape:
escape = false
p[j] = b
j++
case b == '\\':
escape = true
case b == '"':
return string(p[:j]), s[i+1:]
default:
p[j] = b
j++
}
}
return "", ""
}
}
return "", ""
}
......@@ -50,19 +50,22 @@ func newCall(t TestHelper, receiver interface{}, method string, methodType refle
t.Helper()
// TODO: check arity, types.
margs := make([]Matcher, len(args))
mArgs := make([]Matcher, len(args))
for i, arg := range args {
if m, ok := arg.(Matcher); ok {
margs[i] = m
mArgs[i] = m
} else if arg == nil {
// Handle nil specially so that passing a nil interface value
// will match the typed nils of concrete args.
margs[i] = Nil()
mArgs[i] = Nil()
} else {
margs[i] = Eq(arg)
mArgs[i] = Eq(arg)
}
}
// callerInfo's skip should be updated if the number of calls between the user's test
// and this line changes, i.e. this code is wrapped in another anonymous function.
// 0 is us, 1 is RecordCallWithMethodType(), 2 is the generated recorder, and 3 is the user's test.
origin := callerInfo(3)
actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} {
// Synthesize the zero value for each of the return args' types.
......@@ -73,7 +76,7 @@ func newCall(t TestHelper, receiver interface{}, method string, methodType refle
return rets
}}
return &Call{t: t, receiver: receiver, method: method, methodType: methodType,
args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
args: mArgs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
}
// AnyTimes allows the expectation to be called 0 or more times
......@@ -110,19 +113,25 @@ func (c *Call) DoAndReturn(f interface{}) *Call {
v := reflect.ValueOf(f)
c.addAction(func(args []interface{}) []interface{} {
vargs := make([]reflect.Value, len(args))
c.t.Helper()
vArgs := make([]reflect.Value, len(args))
ft := v.Type()
if c.methodType.NumIn() != ft.NumIn() {
c.t.Fatalf("wrong number of arguments in DoAndReturn func for %T.%v: got %d, want %d [%s]",
c.receiver, c.method, ft.NumIn(), c.methodType.NumIn(), c.origin)
return nil
}
for i := 0; i < len(args); i++ {
if args[i] != nil {
vargs[i] = reflect.ValueOf(args[i])
vArgs[i] = reflect.ValueOf(args[i])
} else {
// Use the zero value for the arg.
vargs[i] = reflect.Zero(ft.In(i))
vArgs[i] = reflect.Zero(ft.In(i))
}
}
vrets := v.Call(vargs)
rets := make([]interface{}, len(vrets))
for i, ret := range vrets {
vRets := v.Call(vArgs)
rets := make([]interface{}, len(vRets))
for i, ret := range vRets {
rets[i] = ret.Interface()
}
return rets
......@@ -139,17 +148,23 @@ func (c *Call) Do(f interface{}) *Call {
v := reflect.ValueOf(f)
c.addAction(func(args []interface{}) []interface{} {
vargs := make([]reflect.Value, len(args))
c.t.Helper()
if c.methodType.NumIn() != v.Type().NumIn() {
c.t.Fatalf("wrong number of arguments in Do func for %T.%v: got %d, want %d [%s]",
c.receiver, c.method, v.Type().NumIn(), c.methodType.NumIn(), c.origin)
return nil
}
vArgs := make([]reflect.Value, len(args))
ft := v.Type()
for i := 0; i < len(args); i++ {
if args[i] != nil {
vargs[i] = reflect.ValueOf(args[i])
vArgs[i] = reflect.ValueOf(args[i])
} else {
// Use the zero value for the arg.
vargs[i] = reflect.Zero(ft.In(i))
vArgs[i] = reflect.Zero(ft.In(i))
}
}
v.Call(vargs)
v.Call(vArgs)
return nil
})
return c
......@@ -301,14 +316,9 @@ func (c *Call) matches(args []interface{}) error {
for i, m := range c.args {
if !m.Matches(args[i]) {
got := fmt.Sprintf("%v", args[i])
if gs, ok := m.(GotFormatter); ok {
got = gs.Got(args[i])
}
return fmt.Errorf(
"expected call at %s doesn't match the argument at index %d.\nGot: %v\nWant: %v",
c.origin, i, got, m,
c.origin, i, formatGottenArg(m, args[i]), m,
)
}
}
......@@ -331,7 +341,7 @@ func (c *Call) matches(args []interface{}) error {
// Non-variadic args
if !m.Matches(args[i]) {
return fmt.Errorf("expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
c.origin, strconv.Itoa(i), args[i], m)
c.origin, strconv.Itoa(i), formatGottenArg(m, args[i]), m)
}
continue
}
......@@ -355,12 +365,12 @@ func (c *Call) matches(args []interface{}) error {
// matches all the remaining arguments or the lack of any.
// Convert the remaining arguments, if any, into a slice of the
// expected type.
vargsType := c.methodType.In(c.methodType.NumIn() - 1)
vargs := reflect.MakeSlice(vargsType, 0, len(args)-i)
vArgsType := c.methodType.In(c.methodType.NumIn() - 1)
vArgs := reflect.MakeSlice(vArgsType, 0, len(args)-i)
for _, arg := range args[i:] {
vargs = reflect.Append(vargs, reflect.ValueOf(arg))
vArgs = reflect.Append(vArgs, reflect.ValueOf(arg))
}
if m.Matches(vargs.Interface()) {
if m.Matches(vArgs.Interface()) {
// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any())
// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher)
// Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any())
......@@ -373,16 +383,16 @@ func (c *Call) matches(args []interface{}) error {
// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE)
// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD)
// Got Foo(a, b, c) want Foo(matcherA, matcherB)
return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
c.origin, strconv.Itoa(i), args[i:], c.args[i])
return fmt.Errorf("expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
c.origin, strconv.Itoa(i), formatGottenArg(m, args[i:]), c.args[i])
}
}
// Check that all prerequisite calls have been satisfied.
for _, preReqCall := range c.preReqs {
if !preReqCall.satisfied() {
return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
return fmt.Errorf("expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
c.origin, preReqCall, c)
}
}
......@@ -425,3 +435,11 @@ func setSlice(arg interface{}, v reflect.Value) {
func (c *Call) addAction(action func([]interface{}) []interface{}) {
c.actions = append(c.actions, action)
}
func formatGottenArg(m Matcher, arg interface{}) string {
got := fmt.Sprintf("%v (%T)", arg, arg)
if gs, ok := m.(GotFormatter); ok {
got = gs.Got(arg)
}
return got
}
......@@ -16,6 +16,7 @@ package gomock
import (
"bytes"
"errors"
"fmt"
)
......@@ -84,14 +85,18 @@ func (cs callSet) FindMatch(receiver interface{}, method string, args []interfac
for _, call := range exhausted {
if err := call.matches(args); err != nil {
_, _ = fmt.Fprintf(&callsErrors, "\n%v", err)
continue
}
_, _ = fmt.Fprintf(
&callsErrors, "all expected calls for method %q have been exhausted", method,
)
}
if len(expected)+len(exhausted) == 0 {
_, _ = fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method)
}
return nil, fmt.Errorf(callsErrors.String())
return nil, errors.New(callsErrors.String())
}
// Failures returns the calls that are not satisfied.
......
......@@ -50,9 +50,6 @@
// mockObj.EXPECT().SomeMethod(2, "second"),
// mockObj.EXPECT().SomeMethod(3, "third"),
// )
//
// TODO:
// - Handle different argument/return types (e.g. ..., chan, map, interface).
package gomock
import (
......@@ -77,6 +74,15 @@ type TestHelper interface {
Helper()
}
// cleanuper is used to check if TestHelper also has the `Cleanup` method. A
// common pattern is to pass in a `*testing.T` to
// `NewController(t TestReporter)`. In Go 1.14+, `*testing.T` has a cleanup
// method. This can be utilized to call `Finish()` so the caller of this library
// does not have to.
type cleanuper interface {
Cleanup(func())
}
// A Controller represents the top-level control of a mock ecosystem. It
// defines the scope and lifetime of mock objects, as well as their
// expectations. It is safe to call Controller's methods from multiple
......@@ -115,29 +121,43 @@ type Controller struct {
// NewController returns a new Controller. It is the preferred way to create a
// Controller.
//
// New in go1.14+, if you are passing a *testing.T into this function you no
// longer need to call ctrl.Finish() in your test methods.
func NewController(t TestReporter) *Controller {
h, ok := t.(TestHelper)
if !ok {
h = nopTestHelper{t}
h = &nopTestHelper{t}
}
return &Controller{
ctrl := &Controller{
T: h,
expectedCalls: newCallSet(),
}
if c, ok := isCleanuper(ctrl.T); ok {
c.Cleanup(func() {
ctrl.T.Helper()
ctrl.finish(true, nil)
})
}
return ctrl
}
type cancelReporter struct {
TestHelper
t TestHelper
cancel func()
}
func (r *cancelReporter) Errorf(format string, args ...interface{}) {
r.TestHelper.Errorf(format, args...)
r.t.Errorf(format, args...)
}
func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
defer r.cancel()
r.TestHelper.Fatalf(format, args...)
r.t.Fatalf(format, args...)
}
func (r *cancelReporter) Helper() {
r.t.Helper()
}
// WithContext returns a new Controller and a Context, which is cancelled on any
......@@ -145,15 +165,22 @@ func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) {
h, ok := t.(TestHelper)
if !ok {
h = nopTestHelper{t}
h = &nopTestHelper{t: t}
}
ctx, cancel := context.WithCancel(ctx)
return NewController(&cancelReporter{h, cancel}), ctx
return NewController(&cancelReporter{t: h, cancel: cancel}), ctx
}
type nopTestHelper struct {
TestReporter
t TestReporter
}
func (h *nopTestHelper) Errorf(format string, args ...interface{}) {
h.t.Errorf(format, args...)
}
func (h *nopTestHelper) Fatalf(format string, args ...interface{}) {
h.t.Fatalf(format, args...)
}
func (h nopTestHelper) Helper() {}
......@@ -197,7 +224,10 @@ func (ctrl *Controller) Call(receiver interface{}, method string, args ...interf
expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args)
if err != nil {
origin := callerInfo(2)
// callerInfo's skip should be updated if the number of calls between the user's test
// and this line changes, i.e. this code is wrapped in another anonymous function.
// 0 is us, 1 is controller.Call(), 2 is the generated mock, and 3 is the user's test.
origin := callerInfo(3)
ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err)
}
......@@ -229,21 +259,33 @@ func (ctrl *Controller) Call(receiver interface{}, method string, args ...interf
// Finish checks to see if all the methods that were expected to be called
// were called. It should be invoked for each Controller. It is not idempotent
// and therefore can only be invoked once.
//
// New in go1.14+, if you are passing a *testing.T into NewController function you no
// longer need to call ctrl.Finish() in your test methods.
func (ctrl *Controller) Finish() {
// If we're currently panicking, probably because this is a deferred call.
// This must be recovered in the deferred function.
err := recover()
ctrl.finish(false, err)
}
func (ctrl *Controller) finish(cleanup bool, panicErr interface{}) {
ctrl.T.Helper()
ctrl.mu.Lock()
defer ctrl.mu.Unlock()
if ctrl.finished {
ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
if _, ok := isCleanuper(ctrl.T); !ok {
ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
}
return
}
ctrl.finished = true
// If we're currently panicking, probably because this is a deferred call,
// pass through the panic.
if err := recover(); err != nil {
panic(err)
// Short-circuit, pass through the panic.
if panicErr != nil {
panic(panicErr)
}
// Check that all remaining expected calls are satisfied.
......@@ -252,13 +294,43 @@ func (ctrl *Controller) Finish() {
ctrl.T.Errorf("missing call(s) to %v", call)
}
if len(failures) != 0 {
ctrl.T.Fatalf("aborting test due to missing call(s)")
if !cleanup {
ctrl.T.Fatalf("aborting test due to missing call(s)")
return
}
ctrl.T.Errorf("aborting test due to missing call(s)")
}
}
// callerInfo returns the file:line of the call site. skip is the number
// of stack frames to skip when reporting. 0 is callerInfo's call site.
func callerInfo(skip int) string {
if _, file, line, ok := runtime.Caller(skip + 1); ok {
return fmt.Sprintf("%s:%d", file, line)
}
return "unknown file"
}
// isCleanuper checks it if t's base TestReporter has a Cleanup method.
func isCleanuper(t TestReporter) (cleanuper, bool) {
tr := unwrapTestReporter(t)
c, ok := tr.(cleanuper)
return c, ok
}
// unwrapTestReporter unwraps TestReporter to the base implementation.
func unwrapTestReporter(t TestReporter) TestReporter {
tr := t
switch nt := t.(type) {
case *cancelReporter:
tr = nt.t
if h, check := tr.(*nopTestHelper); check {
tr = h.t
}
case *nopTestHelper:
tr = nt.t
default:
// not wrapped
}
return tr
}
......@@ -102,11 +102,25 @@ type eqMatcher struct {
}
func (e eqMatcher) Matches(x interface{}) bool {
return reflect.DeepEqual(e.x, x)
// In case, some value is nil
if e.x == nil || x == nil {
return reflect.DeepEqual(e.x, x)
}
// Check if types assignable and convert them to common type
x1Val := reflect.ValueOf(e.x)
x2Val := reflect.ValueOf(x)
if x1Val.Type().AssignableTo(x2Val.Type()) {
x1ValConverted := x1Val.Convert(x2Val.Type())
return reflect.DeepEqual(x1ValConverted.Interface(), x2Val.Interface())
}
return false
}
func (e eqMatcher) String() string {
return fmt.Sprintf("is equal to %v", e.x)
return fmt.Sprintf("is equal to %v (%T)", e.x, e.x)
}
type nilMatcher struct{}
......@@ -139,7 +153,6 @@ func (n notMatcher) Matches(x interface{}) bool {
}
func (n notMatcher) String() string {
// TODO: Improve this if we add a NotString method to the Matcher interface.
return "not(" + n.m.String() + ")"
}
......@@ -194,6 +207,70 @@ func (m lenMatcher) String() string {
return fmt.Sprintf("has length %d", m.i)
}
type inAnyOrderMatcher struct {
x interface{}
}
func (m inAnyOrderMatcher) Matches(x interface{}) bool {
given, ok := m.prepareValue(x)
if !ok {
return false
}
wanted, ok := m.prepareValue(m.x)
if !ok {
return false
}
if given.Len() != wanted.Len() {
return false
}
usedFromGiven := make([]bool, given.Len())
foundFromWanted := make([]bool, wanted.Len())
for i := 0; i < wanted.Len(); i++ {
wantedMatcher := Eq(wanted.Index(i).Interface())
for j := 0; j < given.Len(); j++ {
if usedFromGiven[j] {
continue
}
if wantedMatcher.Matches(given.Index(j).Interface()) {
foundFromWanted[i] = true
usedFromGiven[j] = true
break
}
}
}
missingFromWanted := 0
for _, found := range foundFromWanted {
if !found {
missingFromWanted++
}
}
extraInGiven := 0
for _, used := range usedFromGiven {
if !used {
extraInGiven++
}
}
return extraInGiven == 0 && missingFromWanted == 0
}
func (m inAnyOrderMatcher) prepareValue(x interface{}) (reflect.Value, bool) {
xValue := reflect.ValueOf(x)
switch xValue.Kind() {
case reflect.Slice, reflect.Array:
return xValue, true
default:
return reflect.Value{}, false
}
}
func (m inAnyOrderMatcher) String() string {
return fmt.Sprintf("has the same elements as %v", m.x)
}
// Constructors
// All returns a composite Matcher that returns true if and only all of the
......@@ -245,7 +322,7 @@ func Not(x interface{}) Matcher {
// AssignableToTypeOf(s).Matches(time.Second) // returns true
// AssignableToTypeOf(s).Matches(99) // returns false
//
// var ctx = reflect.TypeOf((*context.Context)).Elem()
// var ctx = reflect.TypeOf((*context.Context)(nil)).Elem()
// AssignableToTypeOf(ctx).Matches(context.Background()) // returns true
func AssignableToTypeOf(x interface{}) Matcher {
if xt, ok := x.(reflect.Type); ok {
......@@ -253,3 +330,12 @@ func AssignableToTypeOf(x interface{}) Matcher {
}
return assignableToTypeOfMatcher{reflect.TypeOf(x)}
}
// InAnyOrder is a Matcher that returns true for collections of the same elements ignoring the order.
//
// Example usage:
// InAnyOrder([]int{1, 2, 3}).Matches([]int{1, 3, 2}) // returns true
// InAnyOrder([]int{1, 2, 3}).Matches([]int{1, 2}) // returns false
func InAnyOrder(x interface{}) Matcher {
return inAnyOrderMatcher{x}
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package and provides helpers for adding Close to io.{Reader|Writer}.
package and
import (
"io"
)
// ReadCloser implements io.ReadCloser by reading from a particular io.Reader
// and then calling the provided "Close()" method.
type ReadCloser struct {
io.Reader
CloseFunc func() error
}
var _ io.ReadCloser = (*ReadCloser)(nil)
// Close implements io.ReadCloser
func (rac *ReadCloser) Close() error {
return rac.CloseFunc()
}
// WriteCloser implements io.WriteCloser by reading from a particular io.Writer
// and then calling the provided "Close()" method.
type WriteCloser struct {
io.Writer
CloseFunc func() error
}
var _ io.WriteCloser = (*WriteCloser)(nil)
// Close implements io.WriteCloser
func (wac *WriteCloser) Close() error {
return wac.CloseFunc()
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package estargz adapts the containerd estargz package to our abstractions.
package estargz
import (
"bytes"
"io"
"io/ioutil"
"github.com/containerd/stargz-snapshotter/estargz"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// Assert that what we're returning is an io.ReadCloser
var _ io.ReadCloser = (*estargz.Blob)(nil)
// ReadCloser reads uncompressed tarball input from the io.ReadCloser and
// returns:
// * An io.ReadCloser from which compressed data may be read, and
// * A v1.Hash with the hash of the estargz table of contents, or
// * An error if the estargz processing encountered a problem.
//
// Refer to estargz for the options:
// https://pkg.go.dev/github.com/containerd/stargz-snapshotter/estargz@v0.4.1#Option
func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) {
defer r.Close()
// TODO(#876): Avoid buffering into memory.
bs, err := ioutil.ReadAll(r)
if err != nil {
return nil, v1.Hash{}, err
}
br := bytes.NewReader(bs)
rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...)
if err != nil {
return nil, v1.Hash{}, err
}
h, err := v1.NewHash(rc.TOCDigest().String())
return rc, h, err
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gzip provides helper functions for interacting with gzipped streams.
package gzip
import (
"bufio"
"bytes"
"compress/gzip"
"io"
"github.com/google/go-containerregistry/internal/and"
)
var gzipMagicHeader = []byte{'\x1f', '\x8b'}
// ReadCloser reads uncompressed input data from the io.ReadCloser and
// returns an io.ReadCloser from which compressed data may be read.
// This uses gzip.BestSpeed for the compression level.
func ReadCloser(r io.ReadCloser) io.ReadCloser {
return ReadCloserLevel(r, gzip.BestSpeed)
}
// ReadCloserLevel reads uncompressed input data from the io.ReadCloser and
// returns an io.ReadCloser from which compressed data may be read.
// Refer to compress/gzip for the level:
// https://golang.org/pkg/compress/gzip/#pkg-constants
func ReadCloserLevel(r io.ReadCloser, level int) io.ReadCloser {
pr, pw := io.Pipe()
// For highly compressible layers, gzip.Writer will output a very small
// number of bytes per Write(). This is normally fine, but when pushing
// to a registry, we want to ensure that we're taking full advantage of
// the available bandwidth instead of sending tons of tiny writes over
// the wire.
// 64K ought to be small enough for anybody.
bw := bufio.NewWriterSize(pw, 2<<16)
// Returns err so we can pw.CloseWithError(err)
go func() error {
// TODO(go1.14): Just defer {pw,gw,r}.Close like you'd expect.
// Context: https://golang.org/issue/24283
gw, err := gzip.NewWriterLevel(bw, level)
if err != nil {
return pw.CloseWithError(err)
}
if _, err := io.Copy(gw, r); err != nil {
defer r.Close()
defer gw.Close()
return pw.CloseWithError(err)
}
// Close gzip writer to Flush it and write gzip trailers.
if err := gw.Close(); err != nil {
return pw.CloseWithError(err)
}
// Flush bufio writer to ensure we write out everything.
if err := bw.Flush(); err != nil {
return pw.CloseWithError(err)
}
// We don't really care if these fail.
defer pw.Close()
defer r.Close()
return nil
}()
return pr
}
// UnzipReadCloser reads compressed input data from the io.ReadCloser and
// returns an io.ReadCloser from which uncompessed data may be read.
func UnzipReadCloser(r io.ReadCloser) (io.ReadCloser, error) {
gr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
return &and.ReadCloser{
Reader: gr,
CloseFunc: func() error {
// If the unzip fails, then this seems to return the same
// error as the read. We don't want this to interfere with
// us closing the main ReadCloser, since this could leave
// an open file descriptor (fails on Windows).
gr.Close()
return r.Close()
},
}, nil
}
// Is detects whether the input stream is compressed.
func Is(r io.Reader) (bool, error) {
magicHeader := make([]byte, 2)
n, err := r.Read(magicHeader)
if n == 0 && err == io.EOF {
return false, nil
}
if err != nil {
return false, err
}
return bytes.Equal(magicHeader, gzipMagicHeader), nil
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package redact contains a simple context signal for redacting requests.
package redact
import (
"context"
)
type contextKey string
var redactKey = contextKey("redact")
// NewContext creates a new ctx with the reason for redaction.
func NewContext(ctx context.Context, reason string) context.Context {
return context.WithValue(ctx, redactKey, reason)
}
// FromContext returns the redaction reason, if any.
func FromContext(ctx context.Context) (bool, string) {
reason, ok := ctx.Value(redactKey).(string)
return ok, reason
}
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package retry provides methods for retrying operations. It is a thin wrapper
// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier.
package retry
import (
"context"
"fmt"
"github.com/google/go-containerregistry/internal/retry/wait"
)
// Backoff is an alias of our own wait.Backoff to avoid name conflicts with
// the kubernetes wait package. Typing retry.Backoff is aesier than fixing
// the wrong import every time you use wait.Backoff.
type Backoff = wait.Backoff
// This is implemented by several errors in the net package as well as our
// transport.Error.
type temporary interface {
Temporary() bool
}
// IsTemporary returns true if err implements Temporary() and it returns true.
func IsTemporary(err error) bool {
if err == context.DeadlineExceeded {
return false
}
if te, ok := err.(temporary); ok && te.Temporary() {
return true
}
return false
}
// IsNotNil returns true if err is not nil.
func IsNotNil(err error) bool {
return err != nil
}
// Predicate determines whether an error should be retried.
type Predicate func(error) (retry bool)
// Retry retries a given function, f, until a predicate is satisfied, using
// exponential backoff. If the predicate is never satisfied, it will return the
// last error returned by f.
func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) {
if f == nil {
return fmt.Errorf("nil f passed to retry")
}
if p == nil {
return fmt.Errorf("nil p passed to retry")
}
condition := func() (bool, error) {
err = f()
if p(err) {
return false, nil
}
return true, err
}
wait.ExponentialBackoff(backoff, condition)
return
}
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package wait is a subset of k8s.io/apimachinery to avoid conflicts
// in dependencies (specifically, logging).
package wait
import (
"errors"
"math/rand"
"time"
)
// Jitter returns a time.Duration between duration and duration + maxFactor *
// duration.
//
// This allows clients to avoid converging on periodic behavior. If maxFactor
// is 0.0, a suggested default value will be chosen.
func Jitter(duration time.Duration, maxFactor float64) time.Duration {
if maxFactor <= 0.0 {
maxFactor = 1.0
}
wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))
return wait
}
// ErrWaitTimeout is returned when the condition exited without success.
var ErrWaitTimeout = errors.New("timed out waiting for the condition")
// ConditionFunc returns true if the condition is satisfied, or an error
// if the loop should be aborted.
type ConditionFunc func() (done bool, err error)
// Backoff holds parameters applied to a Backoff function.
type Backoff struct {
// The initial duration.
Duration time.Duration
// Duration is multiplied by factor each iteration, if factor is not zero
// and the limits imposed by Steps and Cap have not been reached.
// Should not be negative.
// The jitter does not contribute to the updates to the duration parameter.
Factor float64
// The sleep at each iteration is the duration plus an additional
// amount chosen uniformly at random from the interval between
// zero and `jitter*duration`.
Jitter float64
// The remaining number of iterations in which the duration
// parameter may change (but progress can be stopped earlier by
// hitting the cap). If not positive, the duration is not
// changed. Used for exponential backoff in combination with
// Factor and Cap.
Steps int
// A limit on revised values of the duration parameter. If a
// multiplication by the factor parameter would make the duration
// exceed the cap then the duration is set to the cap and the
// steps parameter is set to zero.
Cap time.Duration
}
// Step (1) returns an amount of time to sleep determined by the
// original Duration and Jitter and (2) mutates the provided Backoff
// to update its Steps and Duration.
func (b *Backoff) Step() time.Duration {
if b.Steps < 1 {
if b.Jitter > 0 {
return Jitter(b.Duration, b.Jitter)
}
return b.Duration
}
b.Steps--
duration := b.Duration
// calculate the next step
if b.Factor != 0 {
b.Duration = time.Duration(float64(b.Duration) * b.Factor)
if b.Cap > 0 && b.Duration > b.Cap {
b.Duration = b.Cap
b.Steps = 0
}
}
if b.Jitter > 0 {
duration = Jitter(duration, b.Jitter)
}
return duration
}
// ExponentialBackoff repeats a condition check with exponential backoff.
//
// It repeatedly checks the condition and then sleeps, using `backoff.Step()`
// to determine the length of the sleep and adjust Duration and Steps.
// Stops and returns as soon as:
// 1. the condition check returns true or an error,
// 2. `backoff.Steps` checks of the condition have been done, or
// 3. a sleep truncated by the cap on duration has been completed.
// In case (1) the returned error is what the condition function returned.
// In all other cases, ErrWaitTimeout is returned.
func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error {
for backoff.Steps > 0 {
if ok, err := condition(); err != nil || ok {
return err
}
if backoff.Steps == 1 {
break
}
time.Sleep(backoff.Step())
}
return ErrWaitTimeout
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package verify provides a ReadCloser that verifies content matches the
// expected hash values.
package verify
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"github.com/google/go-containerregistry/internal/and"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// SizeUnknown is a sentinel value to indicate that the expected size is not known.
const SizeUnknown = -1
type verifyReader struct {
inner io.Reader
hasher hash.Hash
expected v1.Hash
gotSize, wantSize int64
}
// Read implements io.Reader
func (vc *verifyReader) Read(b []byte) (int, error) {
n, err := vc.inner.Read(b)
vc.gotSize += int64(n)
if err == io.EOF {
if vc.wantSize != SizeUnknown && vc.gotSize != vc.wantSize {
return n, fmt.Errorf("error verifying size; got %d, want %d", vc.gotSize, vc.wantSize)
}
got := hex.EncodeToString(vc.hasher.Sum(make([]byte, 0, vc.hasher.Size())))
if want := vc.expected.Hex; got != want {
return n, fmt.Errorf("error verifying %s checksum after reading %d bytes; got %q, want %q",
vc.expected.Algorithm, vc.gotSize, got, want)
}
}
return n, err
}
// ReadCloser wraps the given io.ReadCloser to verify that its contents match
// the provided v1.Hash before io.EOF is returned.
//
// The reader will only be read up to size bytes, to prevent resource
// exhaustion. If EOF is returned before size bytes are read, an error is
// returned.
//
// A size of SizeUnknown (-1) indicates disables size verification when the size
// is unknown ahead of time.
func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) {
w, err := v1.Hasher(h.Algorithm)
if err != nil {
return nil, err
}
var r2 io.Reader = r
if size != SizeUnknown {
r2 = io.LimitReader(io.TeeReader(r, w), size)
}
return &and.ReadCloser{
Reader: &verifyReader{
inner: r2,
hasher: w,
expected: h,
wantSize: size,
},
CloseFunc: r.Close,
}, nil
}
// Descriptor verifies that the embedded Data field matches the Size and Digest
// fields of the given v1.Descriptor, returning an error if the Data field is
// missing or if it contains incorrect data.
func Descriptor(d v1.Descriptor) error {
if d.Data == nil {
return errors.New("error verifying descriptor; Data == nil")
}
h, sz, err := v1.SHA256(bytes.NewReader(d.Data))
if err != nil {
return err
}
if h != d.Digest {
return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest)
}
if sz != d.Size {
return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size)
}
return nil
}
......@@ -16,6 +16,7 @@ package authn
import (
"os"
"sync"
"github.com/docker/cli/cli/config"
"github.com/docker/cli/cli/config/types"
......@@ -42,7 +43,9 @@ type Keychain interface {
// defaultKeychain implements Keychain with the semantics of the standard Docker
// credential keychain.
type defaultKeychain struct{}
type defaultKeychain struct {
mu sync.Mutex
}
var (
// DefaultKeychain implements Keychain by interpreting the docker config file.
......@@ -57,6 +60,8 @@ const (
// Resolve implements Keychain.
func (dk *defaultKeychain) Resolve(target Resource) (Authenticator, error) {
dk.mu.Lock()
defer dk.mu.Unlock()
cf, err := config.Load(os.Getenv("DOCKER_CONFIG"))
if err != nil {
return nil, err
......
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logs exposes the loggers used by this library.
package logs
import (
"io/ioutil"
"log"
)
var (
// Warn is used to log non-fatal errors.
Warn = log.New(ioutil.Discard, "", log.LstdFlags)
// Progress is used to log notable, successful events.
Progress = log.New(ioutil.Discard, "", log.LstdFlags)
// Debug is used to log information that is useful for debugging.
Debug = log.New(ioutil.Discard, "", log.LstdFlags)
)
// Enabled checks to see if the logger's writer is set to something other
// than ioutil.Discard. This allows callers to avoid expensive operations
// that will end up in /dev/null anyway.
func Enabled(l *log.Logger) bool {
return l.Writer() != ioutil.Discard
}
......@@ -35,9 +35,9 @@ func stripRunesFn(runes string) func(rune) rune {
func checkElement(name, element, allowedRunes string, minRunes, maxRunes int) error {
numRunes := utf8.RuneCountInString(element)
if (numRunes < minRunes) || (maxRunes < numRunes) {
return NewErrBadName("%s must be between %d and %d runes in length: %s", name, minRunes, maxRunes, element)
return newErrBadName("%s must be between %d and %d runes in length: %s", name, minRunes, maxRunes, element)
} else if len(strings.Map(stripRunesFn(allowedRunes), element)) != 0 {
return NewErrBadName("%s can only contain the runes `%s`: %s", name, allowedRunes, element)
return newErrBadName("%s can only contain the runes `%s`: %s", name, allowedRunes, element)
}
return nil
}
......@@ -69,7 +69,7 @@ func NewDigest(name string, opts ...Option) (Digest, error) {
// Split on "@"
parts := strings.Split(name, digestDelim)
if len(parts) != 2 {
return Digest{}, NewErrBadName("a digest must contain exactly one '@' separator (e.g. registry/repository@digest) saw: %s", name)
return Digest{}, newErrBadName("a digest must contain exactly one '@' separator (e.g. registry/repository@digest) saw: %s", name)
}
base := parts[0]
digest := parts[1]
......
......@@ -14,7 +14,10 @@
package name
import "fmt"
import (
"errors"
"fmt"
)
// ErrBadName is an error for when a bad docker name is supplied.
type ErrBadName struct {
......@@ -25,13 +28,15 @@ func (e *ErrBadName) Error() string {
return e.info
}
// NewErrBadName returns a ErrBadName which returns the given formatted string from Error().
func NewErrBadName(fmtStr string, args ...interface{}) *ErrBadName {
// newErrBadName returns a ErrBadName which returns the given formatted string from Error().
func newErrBadName(fmtStr string, args ...interface{}) *ErrBadName {
return &ErrBadName{fmt.Sprintf(fmtStr, args...)}
}
// IsErrBadName returns true if the given error is an ErrBadName.
//
// Deprecated: Use errors.Is.
func IsErrBadName(err error) bool {
_, ok := err.(*ErrBadName)
return ok
var berr *ErrBadName
return errors.As(err, &berr)
}
......@@ -44,8 +44,7 @@ func ParseReference(s string, opts ...Option) (Reference, error) {
if d, err := NewDigest(s, opts...); err == nil {
return d, nil
}
return nil, NewErrBadName("could not parse reference: " + s)
return nil, newErrBadName("could not parse reference: " + s)
}
type stringConst string
......
......@@ -98,7 +98,7 @@ func checkRegistry(name string) error {
// Per RFC 3986, registries (authorities) are required to be prefixed with "//"
// url.Host == hostname[:port] == authority
if url, err := url.Parse("//" + name); err != nil || url.Host != name {
return NewErrBadName("registries must be valid RFC 3986 URI authorities: %s", name)
return newErrBadName("registries must be valid RFC 3986 URI authorities: %s", name)
}
return nil
}
......@@ -108,7 +108,7 @@ func checkRegistry(name string) error {
func NewRegistry(name string, opts ...Option) (Registry, error) {
opt := makeOptions(opts...)
if opt.strict && len(name) == 0 {
return Registry{}, NewErrBadName("strict validation requires the registry to be explicitly defined")
return Registry{}, newErrBadName("strict validation requires the registry to be explicitly defined")
}
if err := checkRegistry(name); err != nil {
......
......@@ -72,7 +72,7 @@ func checkRepository(repository string) error {
func NewRepository(name string, opts ...Option) (Repository, error) {
opt := makeOptions(opts...)
if len(name) == 0 {
return Repository{}, NewErrBadName("a repository name must be specified")
return Repository{}, newErrBadName("a repository name must be specified")
}
var registry string
......@@ -95,7 +95,7 @@ func NewRepository(name string, opts ...Option) (Repository, error) {
return Repository{}, err
}
if hasImplicitNamespace(repo, reg) && opt.strict {
return Repository{}, NewErrBadName("strict validation requires the full repository path (missing 'library')")
return Repository{}, newErrBadName("strict validation requires the full repository path (missing 'library')")
}
return Repository{reg, repo}, nil
}
......
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"io"
"time"
)
// ConfigFile is the configuration file that holds the metadata describing
// how to launch a container. See:
// https://github.com/opencontainers/image-spec/blob/master/config.md
//
// docker_version and os.version are not part of the spec but included
// for backwards compatibility.
type ConfigFile struct {
Architecture string `json:"architecture"`
Author string `json:"author,omitempty"`
Container string `json:"container,omitempty"`
Created Time `json:"created,omitempty"`
DockerVersion string `json:"docker_version,omitempty"`
History []History `json:"history,omitempty"`
OS string `json:"os"`
RootFS RootFS `json:"rootfs"`
Config Config `json:"config"`
OSVersion string `json:"os.version,omitempty"`
}
// History is one entry of a list recording how this container image was built.
type History struct {
Author string `json:"author,omitempty"`
Created Time `json:"created,omitempty"`
CreatedBy string `json:"created_by,omitempty"`
Comment string `json:"comment,omitempty"`
EmptyLayer bool `json:"empty_layer,omitempty"`
}
// Time is a wrapper around time.Time to help with deep copying
type Time struct {
time.Time
}
// DeepCopyInto creates a deep-copy of the Time value. The underlying time.Time
// type is effectively immutable in the time API, so it is safe to
// copy-by-assign, despite the presence of (unexported) Pointer fields.
func (t *Time) DeepCopyInto(out *Time) {
*out = *t
}
// RootFS holds the ordered list of file system deltas that comprise the
// container image's root filesystem.
type RootFS struct {
Type string `json:"type"`
DiffIDs []Hash `json:"diff_ids"`
}
// HealthConfig holds configuration settings for the HEALTHCHECK feature.
type HealthConfig struct {
// Test is the test to perform to check that the container is healthy.
// An empty slice means to inherit the default.
// The options are:
// {} : inherit healthcheck
// {"NONE"} : disable healthcheck
// {"CMD", args...} : exec arguments directly
// {"CMD-SHELL", command} : run command with system's default shell
Test []string `json:",omitempty"`
// Zero means to inherit. Durations are expressed as integer nanoseconds.
Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.
StartPeriod time.Duration `json:",omitempty"` // The start period for the container to initialize before the retries starts to count down.
// Retries is the number of consecutive failures needed to consider a container as unhealthy.
// Zero means inherit.
Retries int `json:",omitempty"`
}
// Config is a submessage of the config file described as:
// The execution parameters which SHOULD be used as a base when running
// a container using the image.
// The names of the fields in this message are chosen to reflect the JSON
// payload of the Config as defined here:
// https://git.io/vrAET
// and
// https://github.com/opencontainers/image-spec/blob/master/config.md
type Config struct {
AttachStderr bool `json:"AttachStderr,omitempty"`
AttachStdin bool `json:"AttachStdin,omitempty"`
AttachStdout bool `json:"AttachStdout,omitempty"`
Cmd []string `json:"Cmd,omitempty"`
Healthcheck *HealthConfig `json:"Healthcheck,omitempty"`
Domainname string `json:"Domainname,omitempty"`
Entrypoint []string `json:"Entrypoint,omitempty"`
Env []string `json:"Env,omitempty"`
Hostname string `json:"Hostname,omitempty"`
Image string `json:"Image,omitempty"`
Labels map[string]string `json:"Labels,omitempty"`
OnBuild []string `json:"OnBuild,omitempty"`
OpenStdin bool `json:"OpenStdin,omitempty"`
StdinOnce bool `json:"StdinOnce,omitempty"`
Tty bool `json:"Tty,omitempty"`
User string `json:"User,omitempty"`
Volumes map[string]struct{} `json:"Volumes,omitempty"`
WorkingDir string `json:"WorkingDir,omitempty"`
ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"`
ArgsEscaped bool `json:"ArgsEscaped,omitempty"`
NetworkDisabled bool `json:"NetworkDisabled,omitempty"`
MacAddress string `json:"MacAddress,omitempty"`
StopSignal string `json:"StopSignal,omitempty"`
Shell []string `json:"Shell,omitempty"`
}
// ParseConfigFile parses the io.Reader's contents into a ConfigFile.
func ParseConfigFile(r io.Reader) (*ConfigFile, error) {
cf := ConfigFile{}
if err := json.NewDecoder(r).Decode(&cf); err != nil {
return nil, err
}
return &cf, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +k8s:deepcopy-gen=package
// Package v1 defines structured types for OCI v1 images
package v1
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"io"
"strconv"
"strings"
)
// Hash is an unqualified digest of some content, e.g. sha256:deadbeef
type Hash struct {
// Algorithm holds the algorithm used to compute the hash.
Algorithm string
// Hex holds the hex portion of the content hash.
Hex string
}
// String reverses NewHash returning the string-form of the hash.
func (h Hash) String() string {
return fmt.Sprintf("%s:%s", h.Algorithm, h.Hex)
}
// NewHash validates the input string is a hash and returns a strongly type Hash object.
func NewHash(s string) (Hash, error) {
h := Hash{}
if err := h.parse(s); err != nil {
return Hash{}, err
}
return h, nil
}
// MarshalJSON implements json.Marshaler
func (h Hash) MarshalJSON() ([]byte, error) {
return json.Marshal(h.String())
}
// UnmarshalJSON implements json.Unmarshaler
func (h *Hash) UnmarshalJSON(data []byte) error {
s, err := strconv.Unquote(string(data))
if err != nil {
return err
}
return h.parse(s)
}
// MarshalText implements encoding.TextMarshaler. This is required to use
// v1.Hash as a key in a map when marshalling JSON.
func (h Hash) MarshalText() (text []byte, err error) {
return []byte(h.String()), nil
}
// UnmarshalText implements encoding.TextUnmarshaler. This is required to use
// v1.Hash as a key in a map when unmarshalling JSON.
func (h *Hash) UnmarshalText(text []byte) error {
return h.parse(string(text))
}
// Hasher returns a hash.Hash for the named algorithm (e.g. "sha256")
func Hasher(name string) (hash.Hash, error) {
switch name {
case "sha256":
return sha256.New(), nil
default:
return nil, fmt.Errorf("unsupported hash: %q", name)
}
}
func (h *Hash) parse(unquoted string) error {
parts := strings.Split(unquoted, ":")
if len(parts) != 2 {
return fmt.Errorf("cannot parse hash: %q", unquoted)
}
rest := strings.TrimLeft(parts[1], "0123456789abcdef")
if len(rest) != 0 {
return fmt.Errorf("found non-hex character in hash: %c", rest[0])
}
hasher, err := Hasher(parts[0])
if err != nil {
return err
}
// Compare the hex to the expected size (2 hex characters per byte)
if len(parts[1]) != hasher.Size()*2 {
return fmt.Errorf("wrong number of hex digits for %s: %s", parts[0], parts[1])
}
h.Algorithm = parts[0]
h.Hex = parts[1]
return nil
}
// SHA256 computes the Hash of the provided io.Reader's content.
func SHA256(r io.Reader) (Hash, int64, error) {
hasher := sha256.New()
n, err := io.Copy(hasher, r)
if err != nil {
return Hash{}, 0, err
}
return Hash{
Algorithm: "sha256",
Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))),
}, n, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Image defines the interface for interacting with an OCI v1 image.
type Image interface {
// Layers returns the ordered collection of filesystem layers that comprise this image.
// The order of the list is oldest/base layer first, and most-recent/top layer last.
Layers() ([]Layer, error)
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)
// Size returns the size of the manifest.
Size() (int64, error)
// ConfigName returns the hash of the image's config file, also known as
// the Image ID.
ConfigName() (Hash, error)
// ConfigFile returns this image's config file.
ConfigFile() (*ConfigFile, error)
// RawConfigFile returns the serialized bytes of ConfigFile().
RawConfigFile() ([]byte, error)
// Digest returns the sha256 of this image's manifest.
Digest() (Hash, error)
// Manifest returns this image's Manifest object.
Manifest() (*Manifest, error)
// RawManifest returns the serialized bytes of Manifest()
RawManifest() ([]byte, error)
// LayerByDigest returns a Layer for interacting with a particular layer of
// the image, looking it up by "digest" (the compressed hash).
LayerByDigest(Hash) (Layer, error)
// LayerByDiffID is an analog to LayerByDigest, looking up by "diff id"
// (the uncompressed hash).
LayerByDiffID(Hash) (Layer, error)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"github.com/google/go-containerregistry/pkg/v1/types"
)
// ImageIndex defines the interface for interacting with an OCI image index.
type ImageIndex interface {
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)
// Digest returns the sha256 of this index's manifest.
Digest() (Hash, error)
// Size returns the size of the manifest.
Size() (int64, error)
// IndexManifest returns this image index's manifest object.
IndexManifest() (*IndexManifest, error)
// RawManifest returns the serialized bytes of IndexManifest().
RawManifest() ([]byte, error)
// Image returns a v1.Image that this ImageIndex references.
Image(Hash) (Image, error)
// ImageIndex returns a v1.ImageIndex that this ImageIndex references.
ImageIndex(Hash) (ImageIndex, error)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"io"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Layer is an interface for accessing the properties of a particular layer of a v1.Image
type Layer interface {
// Digest returns the Hash of the compressed layer.
Digest() (Hash, error)
// DiffID returns the Hash of the uncompressed layer.
DiffID() (Hash, error)
// Compressed returns an io.ReadCloser for the compressed layer contents.
Compressed() (io.ReadCloser, error)
// Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
Uncompressed() (io.ReadCloser, error)
// Size returns the compressed size of the Layer.
Size() (int64, error)
// MediaType returns the media type of the Layer.
MediaType() (types.MediaType, error)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"io"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// Manifest represents the OCI image manifest in a structured way.
type Manifest struct {
SchemaVersion int64 `json:"schemaVersion"`
MediaType types.MediaType `json:"mediaType,omitempty"`
Config Descriptor `json:"config"`
Layers []Descriptor `json:"layers"`
Annotations map[string]string `json:"annotations,omitempty"`
}
// IndexManifest represents an OCI image index in a structured way.
type IndexManifest struct {
SchemaVersion int64 `json:"schemaVersion"`
MediaType types.MediaType `json:"mediaType,omitempty"`
Manifests []Descriptor `json:"manifests"`
Annotations map[string]string `json:"annotations,omitempty"`
}
// Descriptor holds a reference from the manifest to one of its constituent elements.
type Descriptor struct {
MediaType types.MediaType `json:"mediaType"`
Size int64 `json:"size"`
Digest Hash `json:"digest"`
Data []byte `json:"data,omitempty"`
URLs []string `json:"urls,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Platform *Platform `json:"platform,omitempty"`
}
// ParseManifest parses the io.Reader's contents into a Manifest.
func ParseManifest(r io.Reader) (*Manifest, error) {
m := Manifest{}
if err := json.NewDecoder(r).Decode(&m); err != nil {
return nil, err
}
return &m, nil
}
// ParseIndexManifest parses the io.Reader's contents into an IndexManifest.
func ParseIndexManifest(r io.Reader) (*IndexManifest, error) {
im := IndexManifest{}
if err := json.NewDecoder(r).Decode(&im); err != nil {
return nil, err
}
return &im, nil
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package match provides functionality for conveniently matching a v1.Descriptor.
package match
import (
v1 "github.com/google/go-containerregistry/pkg/v1"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
)
// Matcher function that is given a v1.Descriptor, and returns whether or
// not it matches a given rule. Can match on anything it wants in the Descriptor.
type Matcher func(desc v1.Descriptor) bool
// Name returns a match.Matcher that matches based on the value of the
// "org.opencontainers.image.ref.name" annotation:
// github.com/opencontainers/image-spec/blob/v1.0.1/annotations.md#pre-defined-annotation-keys
func Name(name string) Matcher {
return Annotation(imagespec.AnnotationRefName, name)
}
// Annotation returns a match.Matcher that matches based on the provided annotation.
func Annotation(key, value string) Matcher {
return func(desc v1.Descriptor) bool {
if desc.Annotations == nil {
return false
}
if aValue, ok := desc.Annotations[key]; ok && aValue == value {
return true
}
return false
}
}
// Platforms returns a match.Matcher that matches on any one of the provided platforms.
// Ignores any descriptors that do not have a platform.
func Platforms(platforms ...v1.Platform) Matcher {
return func(desc v1.Descriptor) bool {
if desc.Platform == nil {
return false
}
for _, platform := range platforms {
if desc.Platform.Equals(platform) {
return true
}
}
return false
}
}
// MediaTypes returns a match.Matcher that matches at least one of the provided media types.
func MediaTypes(mediaTypes ...string) Matcher {
mts := map[string]bool{}
for _, media := range mediaTypes {
mts[media] = true
}
return func(desc v1.Descriptor) bool {
if desc.MediaType == "" {
return false
}
if _, ok := mts[string(desc.MediaType)]; ok {
return true
}
return false
}
}
// Digests returns a match.Matcher that matches at least one of the provided Digests
func Digests(digests ...v1.Hash) Matcher {
digs := map[v1.Hash]bool{}
for _, digest := range digests {
digs[digest] = true
}
return func(desc v1.Descriptor) bool {
_, ok := digs[desc.Digest]
return ok
}
}
# `partial`
[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial)
## Partial Implementations
There are roughly two kinds of image representations: compressed and uncompressed.
The implementations for these kinds of images are almost identical, with the only
major difference being how blobs (config and layers) are fetched. This common
code lives in this package, where you provide a _partial_ implementation of a
compressed or uncompressed image, and you get back a full `v1.Image` implementation.
### Examples
In a registry, blobs are compressed, so it's easiest to implement a `v1.Image` in terms
of compressed layers. `remote.remoteImage` does this by implementing `CompressedImageCore`:
```go
type CompressedImageCore interface {
RawConfigFile() ([]byte, error)
MediaType() (types.MediaType, error)
RawManifest() ([]byte, error)
LayerByDigest(v1.Hash) (CompressedLayer, error)
}
```
In a tarball, blobs are (often) uncompressed, so it's easiest to implement a `v1.Image` in terms
of uncompressed layers. `tarball.uncompressedImage` does this by implementing `UncompressedImageCore`:
```go
type UncompressedImageCore interface {
RawConfigFile() ([]byte, error)
MediaType() (types.MediaType, error)
LayerByDiffID(v1.Hash) (UncompressedLayer, error)
}
```
## Optional Methods
Where possible, we access some information via optional methods as an optimization.
### [`partial.Descriptor`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Descriptor)
There are some properties of a [`Descriptor`](https://github.com/opencontainers/image-spec/blob/master/descriptor.md#properties) that aren't derivable from just image data:
* `MediaType`
* `Platform`
* `URLs`
* `Annotations`
For example, in a `tarball.Image`, there is a `LayerSources` field that contains
an entire layer descriptor with `URLs` information for foreign layers. This
information can be passed through to callers by implementing this optional
`Descriptor` method.
See [`#654`](https://github.com/google/go-containerregistry/pull/654).
### [`partial.UncompressedSize`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#UncompressedSize)
Usually, you don't need to know the uncompressed size of a layer, since that
information isn't stored in a config file (just he sha256 is needed); however,
there are cases where it is very helpful to know the layer size, e.g. when
writing the uncompressed layer into a tarball.
See [`#655`](https://github.com/google/go-containerregistry/pull/655).
### [`partial.Exists`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/partial#Exists)
We generally don't care about the existence of something as granular as a
layer, and would rather ensure all the invariants of an image are upheld via
the `validate` package. However, there are situations where we want to do a
quick smoke test to ensure that the underlying storage engine hasn't been
corrupted by something e.g. deleting files or blobs. Thus, we've exposed an
optional `Exists` method that does an existence check without actually reading
any bytes.
The `remote` package implements this via `HEAD` requests.
The `layout` package implements this via `os.Stat`.
See [`#838`](https://github.com/google/go-containerregistry/pull/838).
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"io"
"github.com/google/go-containerregistry/internal/gzip"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// CompressedLayer represents the bare minimum interface a natively
// compressed layer must implement for us to produce a v1.Layer
type CompressedLayer interface {
// Digest returns the Hash of the compressed layer.
Digest() (v1.Hash, error)
// Compressed returns an io.ReadCloser for the compressed layer contents.
Compressed() (io.ReadCloser, error)
// Size returns the compressed size of the Layer.
Size() (int64, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
}
// compressedLayerExtender implements v1.Image using the compressed base properties.
type compressedLayerExtender struct {
CompressedLayer
}
// Uncompressed implements v1.Layer
func (cle *compressedLayerExtender) Uncompressed() (io.ReadCloser, error) {
r, err := cle.Compressed()
if err != nil {
return nil, err
}
return gzip.UnzipReadCloser(r)
}
// DiffID implements v1.Layer
func (cle *compressedLayerExtender) DiffID() (v1.Hash, error) {
// If our nested CompressedLayer implements DiffID,
// then delegate to it instead.
if wdi, ok := cle.CompressedLayer.(WithDiffID); ok {
return wdi.DiffID()
}
r, err := cle.Uncompressed()
if err != nil {
return v1.Hash{}, err
}
defer r.Close()
h, _, err := v1.SHA256(r)
return h, err
}
// CompressedToLayer fills in the missing methods from a CompressedLayer so that it implements v1.Layer
func CompressedToLayer(ul CompressedLayer) (v1.Layer, error) {
return &compressedLayerExtender{ul}, nil
}
// CompressedImageCore represents the base minimum interface a natively
// compressed image must implement for us to produce a v1.Image.
type CompressedImageCore interface {
ImageCore
// RawManifest returns the serialized bytes of the manifest.
RawManifest() ([]byte, error)
// LayerByDigest is a variation on the v1.Image method, which returns
// a CompressedLayer instead.
LayerByDigest(v1.Hash) (CompressedLayer, error)
}
// compressedImageExtender implements v1.Image by extending CompressedImageCore with the
// appropriate methods computed from the minimal core.
type compressedImageExtender struct {
CompressedImageCore
}
// Assert that our extender type completes the v1.Image interface
var _ v1.Image = (*compressedImageExtender)(nil)
// Digest implements v1.Image
func (i *compressedImageExtender) Digest() (v1.Hash, error) {
return Digest(i)
}
// ConfigName implements v1.Image
func (i *compressedImageExtender) ConfigName() (v1.Hash, error) {
return ConfigName(i)
}
// Layers implements v1.Image
func (i *compressedImageExtender) Layers() ([]v1.Layer, error) {
hs, err := FSLayers(i)
if err != nil {
return nil, err
}
ls := make([]v1.Layer, 0, len(hs))
for _, h := range hs {
l, err := i.LayerByDigest(h)
if err != nil {
return nil, err
}
ls = append(ls, l)
}
return ls, nil
}
// LayerByDigest implements v1.Image
func (i *compressedImageExtender) LayerByDigest(h v1.Hash) (v1.Layer, error) {
cl, err := i.CompressedImageCore.LayerByDigest(h)
if err != nil {
return nil, err
}
return CompressedToLayer(cl)
}
// LayerByDiffID implements v1.Image
func (i *compressedImageExtender) LayerByDiffID(h v1.Hash) (v1.Layer, error) {
h, err := DiffIDToBlob(i, h)
if err != nil {
return nil, err
}
return i.LayerByDigest(h)
}
// ConfigFile implements v1.Image
func (i *compressedImageExtender) ConfigFile() (*v1.ConfigFile, error) {
return ConfigFile(i)
}
// Manifest implements v1.Image
func (i *compressedImageExtender) Manifest() (*v1.Manifest, error) {
return Manifest(i)
}
// Size implements v1.Image
func (i *compressedImageExtender) Size() (int64, error) {
return Size(i)
}
// CompressedToImage fills in the missing methods from a CompressedImageCore so that it implements v1.Image
func CompressedToImage(cic CompressedImageCore) (v1.Image, error) {
return &compressedImageExtender{
CompressedImageCore: cic,
}, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package partial defines methods for building up a v1.Image from
// minimal subsets that are sufficient for defining a v1.Image.
package partial
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"github.com/google/go-containerregistry/pkg/v1/types"
)
// ImageCore is the core set of properties without which we cannot build a v1.Image
type ImageCore interface {
// RawConfigFile returns the serialized bytes of this image's config file.
RawConfigFile() ([]byte, error)
// MediaType of this image's manifest.
MediaType() (types.MediaType, error)
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"fmt"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/match"
)
// FindManifests given a v1.ImageIndex, find the manifests that fit the matcher.
func FindManifests(index v1.ImageIndex, matcher match.Matcher) ([]v1.Descriptor, error) {
// get the actual manifest list
indexManifest, err := index.IndexManifest()
if err != nil {
return nil, fmt.Errorf("unable to get raw index: %w", err)
}
manifests := []v1.Descriptor{}
// try to get the root of our image
for _, manifest := range indexManifest.Manifests {
if matcher(manifest) {
manifests = append(manifests, manifest)
}
}
return manifests, nil
}
// FindImages given a v1.ImageIndex, find the images that fit the matcher. If a Descriptor
// matches the provider Matcher, but the referenced item is not an Image, ignores it.
// Only returns those that match the Matcher and are images.
func FindImages(index v1.ImageIndex, matcher match.Matcher) ([]v1.Image, error) {
matches := []v1.Image{}
manifests, err := FindManifests(index, matcher)
if err != nil {
return nil, err
}
for _, desc := range manifests {
// if it is not an image, ignore it
if !desc.MediaType.IsImage() {
continue
}
img, err := index.Image(desc.Digest)
if err != nil {
return nil, err
}
matches = append(matches, img)
}
return matches, nil
}
// FindIndexes given a v1.ImageIndex, find the indexes that fit the matcher. If a Descriptor
// matches the provider Matcher, but the referenced item is not an Index, ignores it.
// Only returns those that match the Matcher and are indexes.
func FindIndexes(index v1.ImageIndex, matcher match.Matcher) ([]v1.ImageIndex, error) {
matches := []v1.ImageIndex{}
manifests, err := FindManifests(index, matcher)
if err != nil {
return nil, err
}
for _, desc := range manifests {
if !desc.MediaType.IsIndex() {
continue
}
// if it is not an index, ignore it
idx, err := index.ImageIndex(desc.Digest)
if err != nil {
return nil, err
}
matches = append(matches, idx)
}
return matches, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package partial
import (
"bytes"
"io"
"sync"
"github.com/google/go-containerregistry/internal/gzip"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// UncompressedLayer represents the bare minimum interface a natively
// uncompressed layer must implement for us to produce a v1.Layer
type UncompressedLayer interface {
// DiffID returns the Hash of the uncompressed layer.
DiffID() (v1.Hash, error)
// Uncompressed returns an io.ReadCloser for the uncompressed layer contents.
Uncompressed() (io.ReadCloser, error)
// Returns the mediaType for the compressed Layer
MediaType() (types.MediaType, error)
}
// uncompressedLayerExtender implements v1.Image using the uncompressed base properties.
type uncompressedLayerExtender struct {
UncompressedLayer
// Memoize size/hash so that the methods aren't twice as
// expensive as doing this manually.
hash v1.Hash
size int64
hashSizeError error
once sync.Once
}
// Compressed implements v1.Layer
func (ule *uncompressedLayerExtender) Compressed() (io.ReadCloser, error) {
u, err := ule.Uncompressed()
if err != nil {
return nil, err
}
return gzip.ReadCloser(u), nil
}
// Digest implements v1.Layer
func (ule *uncompressedLayerExtender) Digest() (v1.Hash, error) {
ule.calcSizeHash()
return ule.hash, ule.hashSizeError
}
// Size implements v1.Layer
func (ule *uncompressedLayerExtender) Size() (int64, error) {
ule.calcSizeHash()
return ule.size, ule.hashSizeError
}
func (ule *uncompressedLayerExtender) calcSizeHash() {
ule.once.Do(func() {
var r io.ReadCloser
r, ule.hashSizeError = ule.Compressed()
if ule.hashSizeError != nil {
return
}
defer r.Close()
ule.hash, ule.size, ule.hashSizeError = v1.SHA256(r)
})
}
// UncompressedToLayer fills in the missing methods from an UncompressedLayer so that it implements v1.Layer
func UncompressedToLayer(ul UncompressedLayer) (v1.Layer, error) {
return &uncompressedLayerExtender{UncompressedLayer: ul}, nil
}
// UncompressedImageCore represents the bare minimum interface a natively
// uncompressed image must implement for us to produce a v1.Image
type UncompressedImageCore interface {
ImageCore
// LayerByDiffID is a variation on the v1.Image method, which returns
// an UncompressedLayer instead.
LayerByDiffID(v1.Hash) (UncompressedLayer, error)
}
// UncompressedToImage fills in the missing methods from an UncompressedImageCore so that it implements v1.Image.
func UncompressedToImage(uic UncompressedImageCore) (v1.Image, error) {
return &uncompressedImageExtender{
UncompressedImageCore: uic,
}, nil
}
// uncompressedImageExtender implements v1.Image by extending UncompressedImageCore with the
// appropriate methods computed from the minimal core.
type uncompressedImageExtender struct {
UncompressedImageCore
lock sync.Mutex
manifest *v1.Manifest
}
// Assert that our extender type completes the v1.Image interface
var _ v1.Image = (*uncompressedImageExtender)(nil)
// Digest implements v1.Image
func (i *uncompressedImageExtender) Digest() (v1.Hash, error) {
return Digest(i)
}
// Manifest implements v1.Image
func (i *uncompressedImageExtender) Manifest() (*v1.Manifest, error) {
i.lock.Lock()
defer i.lock.Unlock()
if i.manifest != nil {
return i.manifest, nil
}
b, err := i.RawConfigFile()
if err != nil {
return nil, err
}
cfgHash, cfgSize, err := v1.SHA256(bytes.NewReader(b))
if err != nil {
return nil, err
}
m := &v1.Manifest{
SchemaVersion: 2,
MediaType: types.DockerManifestSchema2,
Config: v1.Descriptor{
MediaType: types.DockerConfigJSON,
Size: cfgSize,
Digest: cfgHash,
},
}
ls, err := i.Layers()
if err != nil {
return nil, err
}
m.Layers = make([]v1.Descriptor, len(ls))
for i, l := range ls {
desc, err := Descriptor(l)
if err != nil {
return nil, err
}
m.Layers[i] = *desc
}
i.manifest = m
return i.manifest, nil
}
// RawManifest implements v1.Image
func (i *uncompressedImageExtender) RawManifest() ([]byte, error) {
return RawManifest(i)
}
// Size implements v1.Image
func (i *uncompressedImageExtender) Size() (int64, error) {
return Size(i)
}
// ConfigName implements v1.Image
func (i *uncompressedImageExtender) ConfigName() (v1.Hash, error) {
return ConfigName(i)
}
// ConfigFile implements v1.Image
func (i *uncompressedImageExtender) ConfigFile() (*v1.ConfigFile, error) {
return ConfigFile(i)
}
// Layers implements v1.Image
func (i *uncompressedImageExtender) Layers() ([]v1.Layer, error) {
diffIDs, err := DiffIDs(i)
if err != nil {
return nil, err
}
ls := make([]v1.Layer, 0, len(diffIDs))
for _, h := range diffIDs {
l, err := i.LayerByDiffID(h)
if err != nil {
return nil, err
}
ls = append(ls, l)
}
return ls, nil
}
// LayerByDiffID implements v1.Image
func (i *uncompressedImageExtender) LayerByDiffID(diffID v1.Hash) (v1.Layer, error) {
ul, err := i.UncompressedImageCore.LayerByDiffID(diffID)
if err != nil {
return nil, err
}
return UncompressedToLayer(ul)
}
// LayerByDigest implements v1.Image
func (i *uncompressedImageExtender) LayerByDigest(h v1.Hash) (v1.Layer, error) {
diffID, err := BlobToDiffID(i, h)
if err != nil {
return nil, err
}
return i.LayerByDiffID(diffID)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"sort"
)
// Platform represents the target os/arch for an image.
type Platform struct {
Architecture string `json:"architecture"`
OS string `json:"os"`
OSVersion string `json:"os.version,omitempty"`
OSFeatures []string `json:"os.features,omitempty"`
Variant string `json:"variant,omitempty"`
Features []string `json:"features,omitempty"`
}
// Equals returns true if the given platform is semantically equivalent to this one.
// The order of Features and OSFeatures is not important.
func (p Platform) Equals(o Platform) bool {
return p.OS == o.OS && p.Architecture == o.Architecture && p.Variant == o.Variant && p.OSVersion == o.OSVersion &&
stringSliceEqualIgnoreOrder(p.OSFeatures, o.OSFeatures) && stringSliceEqualIgnoreOrder(p.Features, o.Features)
}
// stringSliceEqual compares 2 string slices and returns if their contents are identical.
func stringSliceEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, elm := range a {
if elm != b[i] {
return false
}
}
return true
}
// stringSliceEqualIgnoreOrder compares 2 string slices and returns if their contents are identical, ignoring order
func stringSliceEqualIgnoreOrder(a, b []string) bool {
if a != nil && b != nil {
sort.Strings(a)
sort.Strings(b)
}
return stringSliceEqual(a, b)
}
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
// Update representation of an update of transfer progress. Some functions
// in this module can take a channel to which updates will be sent while a
// transfer is in progress.
// +k8s:deepcopy-gen=false
type Update struct {
Total int64
Complete int64
Error error
}
# `remote`
[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote)
The `remote` package implements a client for accessing a registry,
per the [OCI distribution spec](https://github.com/opencontainers/distribution-spec/blob/master/spec.md).
It leans heavily on the lower level [`transport`](/pkg/v1/remote/transport) package, which handles the
authentication handshake and structured errors.
## Usage
```go
package main
import (
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
func main() {
ref, err := name.ParseReference("gcr.io/google-containers/pause")
if err != nil {
panic(err)
}
img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
panic(err)
}
// do stuff with img
}
```
## Structure
<p align="center">
<img src="/images/remote.dot.svg" />
</p>
## Background
There are a lot of confusingly similar terms that come up when talking about images in registries.
### Anatomy of an image
In general...
* A tag refers to an image manifest.
* An image manifest references a config file and an orderered list of _compressed_ layers by sha256 digest.
* A config file references an ordered list of _uncompressed_ layers by sha256 digest and contains runtime configuration.
* The sha256 digest of the config file is the [image id](https://github.com/opencontainers/image-spec/blob/master/config.md#imageid) for the image.
For example, an image with two layers would look something like this:
![image anatomy](/images/image-anatomy.dot.svg)
### Anatomy of an index
In the normal case, an [index](https://github.com/opencontainers/image-spec/blob/master/image-index.md) is used to represent a multi-platform image.
This was the original use case for a [manifest
list](https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list).
![image index anatomy](/images/index-anatomy.dot.svg)
It is possible for an index to reference another index, per the OCI
[image-spec](https://github.com/opencontainers/image-spec/blob/master/media-types.md#compatibility-matrix).
In theory, both an image and image index can reference arbitrary things via
[descriptors](https://github.com/opencontainers/image-spec/blob/master/descriptor.md),
e.g. see the [image layout
example](https://github.com/opencontainers/image-spec/blob/master/image-layout.md#index-example),
which references an application/xml file from an image index.
That could look something like this:
![strange image index anatomy](/images/index-anatomy-strange.dot.svg)
Using a recursive index like this might not be possible with all registries,
but this flexibility allows for some interesting applications, e.g. the
[OCI Artifacts](https://github.com/opencontainers/artifacts) effort.
### Anatomy of an image upload
The structure of an image requires a delicate ordering when uploading an image to a registry.
Below is a (slightly simplified) figure that describes how an image is prepared for upload
to a registry and how the data flows between various artifacts:
![upload](/images/upload.dot.svg)
Note that:
* A config file references the uncompressed layer contents by sha256.
* A manifest references the compressed layer contents by sha256 and the size of the layer.
* A manifest references the config file contents by sha256 and the size of the file.
It follows that during an upload, we need to upload layers before the config file,
and we need to upload the config file before the manifest.
Sometimes, we know all of this information ahead of time, (e.g. when copying from remote.Image),
so the ordering is less important.
In other cases, e.g. when using a [`stream.Layer`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/stream#Layer),
we can't compute anything until we have already uploaded the layer, so we need to be careful about ordering.
## Caveats
### schema 1
This package does not support schema 1 images, see [`#377`](https://github.com/google/go-containerregistry/issues/377),
however, it's possible to do _something_ useful with them via [`remote.Get`](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/remote#Get),
which doesn't try to interpret what is returned by the registry.
[`crane.Copy`](https://godoc.org/github.com/google/go-containerregistry/pkg/crane#Copy) takes advantage of this to implement support for copying schema 1 images,
see [here](https://github.com/google/go-containerregistry/blob/main/pkg/internal/legacy/copy.go).
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
type catalog struct {
Repos []string `json:"repositories"`
}
// CatalogPage calls /_catalog, returning the list of repositories on the registry.
func CatalogPage(target name.Registry, last string, n int, options ...Option) ([]string, error) {
o, err := makeOptions(target, options...)
if err != nil {
return nil, err
}
scopes := []string{target.Scope(transport.PullScope)}
tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes)
if err != nil {
return nil, err
}
query := fmt.Sprintf("last=%s&n=%d", url.QueryEscape(last), n)
uri := url.URL{
Scheme: target.Scheme(),
Host: target.RegistryStr(),
Path: "/v2/_catalog",
RawQuery: query,
}
client := http.Client{Transport: tr}
req, err := http.NewRequest(http.MethodGet, uri.String(), nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req.WithContext(o.context))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := transport.CheckError(resp, http.StatusOK); err != nil {
return nil, err
}
var parsed catalog
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return nil, err
}
return parsed.Repos, nil
}
// Catalog calls /_catalog, returning the list of repositories on the registry.
func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]string, error) {
o, err := makeOptions(target, options...)
if err != nil {
return nil, err
}
scopes := []string{target.Scope(transport.PullScope)}
tr, err := transport.NewWithContext(o.context, target, o.auth, o.transport, scopes)
if err != nil {
return nil, err
}
uri := &url.URL{
Scheme: target.Scheme(),
Host: target.RegistryStr(),
Path: "/v2/_catalog",
}
if o.pageSize > 0 {
uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize)
}
client := http.Client{Transport: tr}
// WithContext overrides the ctx passed directly.
if o.context != context.Background() {
ctx = o.context
}
var (
parsed catalog
repoList []string
)
// get responses until there is no next page
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
req, err := http.NewRequest("GET", uri.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if err := transport.CheckError(resp, http.StatusOK); err != nil {
return nil, err
}
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return nil, err
}
if err := resp.Body.Close(); err != nil {
return nil, err
}
repoList = append(repoList, parsed.Repos...)
uri, err = getNextPageURL(resp)
if err != nil {
return nil, err
}
// no next page
if uri == nil {
break
}
}
return repoList, nil
}
package remote
import (
"context"
"fmt"
"net/http"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
// CheckPushPermission returns an error if the given keychain cannot authorize
// a push operation to the given ref.
//
// This can be useful to check whether the caller has permission to push an
// image before doing work to construct the image.
//
// TODO(#412): Remove the need for this method.
func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error {
auth, err := kc.Resolve(ref.Context().Registry)
if err != nil {
return fmt.Errorf("resolving authorization for %v failed: %w", ref.Context().Registry, err)
}
scopes := []string{ref.Scope(transport.PushScope)}
tr, err := transport.New(ref.Context().Registry, auth, t, scopes)
if err != nil {
return fmt.Errorf("creating push check transport for %v failed: %w", ref.Context().Registry, err)
}
// TODO(jasonhall): Against GCR, just doing the token handshake is
// enough, but this doesn't extend to Dockerhub
// (https://github.com/docker/hub-feedback/issues/1771), so we actually
// need to initiate an upload to tell whether the credentials can
// authorize a push. Figure out how to return early here when we can,
// to avoid a roundtrip for spec-compliant registries.
w := writer{
repo: ref.Context(),
client: &http.Client{Transport: tr},
context: context.Background(),
}
loc, _, err := w.initiateUpload("", "")
if loc != "" {
// Since we're only initiating the upload to check whether we
// can, we should attempt to cancel it, in case initiating
// reserves some resources on the server. We shouldn't wait for
// cancelling to complete, and we don't care if it fails.
go w.cancelUpload(loc)
}
return err
}
func (w *writer) cancelUpload(loc string) {
req, err := http.NewRequest(http.MethodDelete, loc, nil)
if err != nil {
return
}
_, _ = w.client.Do(req)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"fmt"
"net/http"
"net/url"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)
// Delete removes the specified image reference from the remote registry.
func Delete(ref name.Reference, options ...Option) error {
o, err := makeOptions(ref.Context(), options...)
if err != nil {
return err
}
scopes := []string{ref.Scope(transport.DeleteScope)}
tr, err := transport.NewWithContext(o.context, ref.Context().Registry, o.auth, o.transport, scopes)
if err != nil {
return err
}
c := &http.Client{Transport: tr}
u := url.URL{
Scheme: ref.Context().Registry.Scheme(),
Host: ref.Context().RegistryStr(),
Path: fmt.Sprintf("/v2/%s/manifests/%s", ref.Context().RepositoryStr(), ref.Identifier()),
}
req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
if err != nil {
return err
}
resp, err := c.Do(req.WithContext(o.context))
if err != nil {
return err
}
defer resp.Body.Close()
return transport.CheckError(resp, http.StatusOK, http.StatusAccepted)
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package remote provides facilities for reading/writing v1.Images from/to
// a remote image registry.
package remote
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"bytes"
"io"
"io/ioutil"
"net/http"
"net/url"
"sync"
"github.com/google/go-containerregistry/internal/redact"
"github.com/google/go-containerregistry/internal/verify"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
"github.com/google/go-containerregistry/pkg/v1/types"
)
var acceptableImageMediaTypes = []types.MediaType{
types.DockerManifestSchema2,
types.OCIManifestSchema1,
}
// remoteImage accesses an image from a remote registry
type remoteImage struct {
fetcher
manifestLock sync.Mutex // Protects manifest
manifest []byte
configLock sync.Mutex // Protects config
config []byte
mediaType types.MediaType
descriptor *v1.Descriptor
}
var _ partial.CompressedImageCore = (*remoteImage)(nil)
// Image provides access to a remote image reference.
func Image(ref name.Reference, options ...Option) (v1.Image, error) {
desc, err := Get(ref, options...)
if err != nil {
return nil, err
}
return desc.Image()
}
func (r *remoteImage) MediaType() (types.MediaType, error) {
if string(r.mediaType) != "" {
return r.mediaType, nil
}
return types.DockerManifestSchema2, nil
}
func (r *remoteImage) RawManifest() ([]byte, error) {
r.manifestLock.Lock()
defer r.manifestLock.Unlock()
if r.manifest != nil {
return r.manifest, nil
}
// NOTE(jonjohnsonjr): We should never get here because the public entrypoints
// do type-checking via remote.Descriptor. I've left this here for tests that
// directly instantiate a remoteImage.
manifest, desc, err := r.fetchManifest(r.Ref, acceptableImageMediaTypes)
if err != nil {
return nil, err
}
if r.descriptor == nil {
r.descriptor = desc
}
r.mediaType = desc.MediaType
r.manifest = manifest
return r.manifest, nil
}
func (r *remoteImage) RawConfigFile() ([]byte, error) {
r.configLock.Lock()
defer r.configLock.Unlock()
if r.config != nil {
return r.config, nil
}
m, err := partial.Manifest(r)
if err != nil {
return nil, err
}
if m.Config.Data != nil {
if err := verify.Descriptor(m.Config); err != nil {
return nil, err
}
r.config = m.Config.Data
return r.config, nil
}
body, err := r.fetchBlob(r.context, m.Config.Size, m.Config.Digest)
if err != nil {
return nil, err
}
defer body.Close()
r.config, err = ioutil.ReadAll(body)
if err != nil {
return nil, err
}
return r.config, nil
}
// Descriptor retains the original descriptor from an index manifest.
// See partial.Descriptor.
func (r *remoteImage) Descriptor() (*v1.Descriptor, error) {
// kind of a hack, but RawManifest does appropriate locking/memoization
// and makes sure r.descriptor is populated.
_, err := r.RawManifest()
return r.descriptor, err
}
// remoteImageLayer implements partial.CompressedLayer
type remoteImageLayer struct {
ri *remoteImage
digest v1.Hash
}
// Digest implements partial.CompressedLayer
func (rl *remoteImageLayer) Digest() (v1.Hash, error) {
return rl.digest, nil
}
// Compressed implements partial.CompressedLayer
func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) {
urls := []url.URL{rl.ri.url("blobs", rl.digest.String())}
// Add alternative layer sources from URLs (usually none).
d, err := partial.BlobDescriptor(rl, rl.digest)
if err != nil {
return nil, err
}
if d.Data != nil {
return verify.ReadCloser(ioutil.NopCloser(bytes.NewReader(d.Data)), d.Size, d.Digest)
}
// We don't want to log binary layers -- this can break terminals.
ctx := redact.NewContext(rl.ri.context, "omitting binary blobs from logs")
for _, s := range d.URLs {
u, err := url.Parse(s)
if err != nil {
return nil, err
}
urls = append(urls, *u)
}
// The lastErr for most pulls will be the same (the first error), but for
// foreign layers we'll want to surface the last one, since we try to pull
// from the registry first, which would often fail.
// TODO: Maybe we don't want to try pulling from the registry first?
var lastErr error
for _, u := range urls {
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
resp, err := rl.ri.Client.Do(req.WithContext(ctx))
if err != nil {
lastErr = err
continue
}
if err := transport.CheckError(resp, http.StatusOK); err != nil {
resp.Body.Close()
lastErr = err
continue
}
return verify.ReadCloser(resp.Body, d.Size, rl.digest)
}
return nil, lastErr
}
// Manifest implements partial.WithManifest so that we can use partial.BlobSize below.
func (rl *remoteImageLayer) Manifest() (*v1.Manifest, error) {
return partial.Manifest(rl.ri)
}
// MediaType implements v1.Layer
func (rl *remoteImageLayer) MediaType() (types.MediaType, error) {
bd, err := partial.BlobDescriptor(rl, rl.digest)
if err != nil {
return "", err
}
return bd.MediaType, nil
}
// Size implements partial.CompressedLayer
func (rl *remoteImageLayer) Size() (int64, error) {
// Look up the size of this digest in the manifest to avoid a request.
return partial.BlobSize(rl, rl.digest)
}
// ConfigFile implements partial.WithManifestAndConfigFile so that we can use partial.BlobToDiffID below.
func (rl *remoteImageLayer) ConfigFile() (*v1.ConfigFile, error) {
return partial.ConfigFile(rl.ri)
}
// DiffID implements partial.WithDiffID so that we don't recompute a DiffID that we already have
// available in our ConfigFile.
func (rl *remoteImageLayer) DiffID() (v1.Hash, error) {
return partial.BlobToDiffID(rl, rl.digest)
}
// Descriptor retains the original descriptor from an image manifest.
// See partial.Descriptor.
func (rl *remoteImageLayer) Descriptor() (*v1.Descriptor, error) {
return partial.BlobDescriptor(rl, rl.digest)
}
// See partial.Exists.
func (rl *remoteImageLayer) Exists() (bool, error) {
return rl.ri.blobExists(rl.digest)
}
// LayerByDigest implements partial.CompressedLayer
func (r *remoteImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) {
return &remoteImageLayer{
ri: r,
digest: h,
}, nil
}
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"bytes"
"fmt"
"sync"
"github.com/google/go-containerregistry/internal/verify"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)
var acceptableIndexMediaTypes = []types.MediaType{
types.DockerManifestList,
types.OCIImageIndex,
}
// remoteIndex accesses an index from a remote registry
type remoteIndex struct {
fetcher
manifestLock sync.Mutex // Protects manifest
manifest []byte
mediaType types.MediaType
descriptor *v1.Descriptor
}
// Index provides access to a remote index reference.
func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) {
desc, err := get(ref, acceptableIndexMediaTypes, options...)
if err != nil {
return nil, err
}
return desc.ImageIndex()
}
func (r *remoteIndex) MediaType() (types.MediaType, error) {
if string(r.mediaType) != "" {
return r.mediaType, nil
}
return types.DockerManifestList, nil
}
func (r *remoteIndex) Digest() (v1.Hash, error) {
return partial.Digest(r)
}
func (r *remoteIndex) Size() (int64, error) {
return partial.Size(r)
}
func (r *remoteIndex) RawManifest() ([]byte, error) {
r.manifestLock.Lock()
defer r.manifestLock.Unlock()
if r.manifest != nil {
return r.manifest, nil
}
// NOTE(jonjohnsonjr): We should never get here because the public entrypoints
// do type-checking via remote.Descriptor. I've left this here for tests that
// directly instantiate a remoteIndex.
manifest, desc, err := r.fetchManifest(r.Ref, acceptableIndexMediaTypes)
if err != nil {
return nil, err
}
if r.descriptor == nil {
r.descriptor = desc
}
r.mediaType = desc.MediaType
r.manifest = manifest
return r.manifest, nil
}
func (r *remoteIndex) IndexManifest() (*v1.IndexManifest, error) {
b, err := r.RawManifest()
if err != nil {
return nil, err
}
return v1.ParseIndexManifest(bytes.NewReader(b))
}
func (r *remoteIndex) Image(h v1.Hash) (v1.Image, error) {
desc, err := r.childByHash(h)
if err != nil {
return nil, err
}
// Descriptor.Image will handle coercing nested indexes into an Image.
return desc.Image()
}
// Descriptor retains the original descriptor from an index manifest.
// See partial.Descriptor.
func (r *remoteIndex) Descriptor() (*v1.Descriptor, error) {
// kind of a hack, but RawManifest does appropriate locking/memoization
// and makes sure r.descriptor is populated.
_, err := r.RawManifest()
return r.descriptor, err
}
func (r *remoteIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) {
desc, err := r.childByHash(h)
if err != nil {
return nil, err
}
return desc.ImageIndex()
}
// Workaround for #819.
func (r *remoteIndex) Layer(h v1.Hash) (v1.Layer, error) {
index, err := r.IndexManifest()
if err != nil {
return nil, err
}
for _, childDesc := range index.Manifests {
if h == childDesc.Digest {
l, err := partial.CompressedToLayer(&remoteLayer{
fetcher: r.fetcher,
digest: h,
})
if err != nil {
return nil, err
}
return &MountableLayer{
Layer: l,
Reference: r.Ref.Context().Digest(h.String()),
}, nil
}
}
return nil, fmt.Errorf("layer not found: %s", h)
}
// Experiment with a better API for v1.ImageIndex. We might want to move this
// to partial?
func (r *remoteIndex) Manifests() ([]partial.Describable, error) {
m, err := r.IndexManifest()
if err != nil {
return nil, err
}
manifests := []partial.Describable{}
for _, desc := range m.Manifests {
switch {
case desc.MediaType.IsImage():
img, err := r.Image(desc.Digest)
if err != nil {
return nil, err
}
manifests = append(manifests, img)
case desc.MediaType.IsIndex():
idx, err := r.ImageIndex(desc.Digest)
if err != nil {
return nil, err
}
manifests = append(manifests, idx)
default:
layer, err := r.Layer(desc.Digest)
if err != nil {
return nil, err
}
manifests = append(manifests, layer)
}
}
return manifests, nil
}
func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) {
desc, err := r.childByPlatform(platform)
if err != nil {
return nil, err
}
// Descriptor.Image will handle coercing nested indexes into an Image.
return desc.Image()
}
// This naively matches the first manifest with matching platform attributes.
//
// We should probably use this instead:
// github.com/containerd/containerd/platforms
//
// But first we'd need to migrate to:
// github.com/opencontainers/image-spec/specs-go/v1
func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) {
index, err := r.IndexManifest()
if err != nil {
return nil, err
}
for _, childDesc := range index.Manifests {
// If platform is missing from child descriptor, assume it's amd64/linux.
p := defaultPlatform
if childDesc.Platform != nil {
p = *childDesc.Platform
}
if matchesPlatform(p, platform) {
return r.childDescriptor(childDesc, platform)
}
}
return nil, fmt.Errorf("no child with platform %+v in index %s", platform, r.Ref)
}
func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) {
index, err := r.IndexManifest()
if err != nil {
return nil, err
}
for _, childDesc := range index.Manifests {
if h == childDesc.Digest {
return r.childDescriptor(childDesc, defaultPlatform)
}
}
return nil, fmt.Errorf("no child with digest %s in index %s", h, r.Ref)
}
// Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option.
func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) {
ref := r.Ref.Context().Digest(child.Digest.String())
var (
manifest []byte
err error
)
if child.Data != nil {
if err := verify.Descriptor(child); err != nil {
return nil, err
}
manifest = child.Data
} else {
manifest, _, err = r.fetchManifest(ref, []types.MediaType{child.MediaType})
if err != nil {
return nil, err
}
}
return &Descriptor{
fetcher: fetcher{
Ref: ref,
Client: r.Client,
context: r.context,
},
Manifest: manifest,
Descriptor: child,
platform: platform,
}, nil
}
// matchesPlatform checks if the given platform matches the required platforms.
// The given platform matches the required platform if
// - architecture and OS are identical.
// - OS version and variant are identical if provided.
// - features and OS features of the required platform are subsets of those of the given platform.
func matchesPlatform(given, required v1.Platform) bool {
// Required fields that must be identical.
if given.Architecture != required.Architecture || given.OS != required.OS {
return false
}
// Optional fields that may be empty, but must be identical if provided.
if required.OSVersion != "" && given.OSVersion != required.OSVersion {
return false
}
if required.Variant != "" && given.Variant != required.Variant {
return false
}
// Verify required platform's features are a subset of given platform's features.
if !isSubset(given.OSFeatures, required.OSFeatures) {
return false
}
if !isSubset(given.Features, required.Features) {
return false
}
return true
}
// isSubset checks if the required array of strings is a subset of the given lst.
func isSubset(lst, required []string) bool {
set := make(map[string]bool)
for _, value := range lst {
set[value] = true
}
for _, value := range required {
if _, ok := set[value]; !ok {
return false
}
}
return true
}
// Copyright 2019 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"io"
"github.com/google/go-containerregistry/internal/redact"
"github.com/google/go-containerregistry/internal/verify"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/types"
)
// remoteImagelayer implements partial.CompressedLayer
type remoteLayer struct {
fetcher
digest v1.Hash
}
// Compressed implements partial.CompressedLayer
func (rl *remoteLayer) Compressed() (io.ReadCloser, error) {
// We don't want to log binary layers -- this can break terminals.
ctx := redact.NewContext(rl.context, "omitting binary blobs from logs")
return rl.fetchBlob(ctx, verify.SizeUnknown, rl.digest)
}
// Compressed implements partial.CompressedLayer
func (rl *remoteLayer) Size() (int64, error) {
resp, err := rl.headBlob(rl.digest)
if err != nil {
return -1, err
}
defer resp.Body.Close()
return resp.ContentLength, nil
}
// Digest implements partial.CompressedLayer
func (rl *remoteLayer) Digest() (v1.Hash, error) {
return rl.digest, nil
}
// MediaType implements v1.Layer
func (rl *remoteLayer) MediaType() (types.MediaType, error) {
return types.DockerLayer, nil
}
// See partial.Exists.
func (rl *remoteLayer) Exists() (bool, error) {
return rl.blobExists(rl.digest)
}
// Layer reads the given blob reference from a registry as a Layer. A blob
// reference here is just a punned name.Digest where the digest portion is the
// digest of the blob to be read and the repository portion is the repo where
// that blob lives.
func Layer(ref name.Digest, options ...Option) (v1.Layer, error) {
o, err := makeOptions(ref.Context(), options...)
if err != nil {
return nil, err
}
f, err := makeFetcher(ref, o)
if err != nil {
return nil, err
}
h, err := v1.NewHash(ref.Identifier())
if err != nil {
return nil, err
}
l, err := partial.CompressedToLayer(&remoteLayer{
fetcher: *f,
digest: h,
})
if err != nil {
return nil, err
}
return &MountableLayer{
Layer: l,
Reference: ref,
}, nil
}
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment