Commit 9fb98a42 authored by Quinton Hoole's avatar Quinton Hoole

Update vendor package google.golang.org/api/googleapi.

parent a0cc59f2
{ {
"ImportPath": "k8s.io/kubernetes", "ImportPath": "k8s.io/kubernetes",
"GoVersion": "go1.6", "GoVersion": "go1.6",
"GodepVersion": "v63", "GodepVersion": "v67",
"Packages": [ "Packages": [
"github.com/ugorji/go/codec/codecgen", "github.com/ugorji/go/codec/codecgen",
"github.com/onsi/ginkgo/ginkgo", "github.com/onsi/ginkgo/ginkgo",
...@@ -2003,27 +2003,27 @@ ...@@ -2003,27 +2003,27 @@
}, },
{ {
"ImportPath": "google.golang.org/api/cloudmonitoring/v2beta2", "ImportPath": "google.golang.org/api/cloudmonitoring/v2beta2",
"Rev": "77e7d383beb96054547729f49c372b3d01e196ff" "Rev": "4300f6b0c8a7f09e521dd0af2cee27e28846e037"
}, },
{ {
"ImportPath": "google.golang.org/api/compute/v1", "ImportPath": "google.golang.org/api/compute/v1",
"Rev": "77e7d383beb96054547729f49c372b3d01e196ff" "Rev": "4300f6b0c8a7f09e521dd0af2cee27e28846e037"
}, },
{ {
"ImportPath": "google.golang.org/api/container/v1", "ImportPath": "google.golang.org/api/container/v1",
"Rev": "77e7d383beb96054547729f49c372b3d01e196ff" "Rev": "4300f6b0c8a7f09e521dd0af2cee27e28846e037"
}, },
{ {
"ImportPath": "google.golang.org/api/gensupport", "ImportPath": "google.golang.org/api/gensupport",
"Rev": "77e7d383beb96054547729f49c372b3d01e196ff" "Rev": "4300f6b0c8a7f09e521dd0af2cee27e28846e037"
}, },
{ {
"ImportPath": "google.golang.org/api/googleapi", "ImportPath": "google.golang.org/api/googleapi",
"Rev": "77e7d383beb96054547729f49c372b3d01e196ff" "Rev": "4300f6b0c8a7f09e521dd0af2cee27e28846e037"
}, },
{ {
"ImportPath": "google.golang.org/api/googleapi/internal/uritemplates", "ImportPath": "google.golang.org/api/googleapi/internal/uritemplates",
"Rev": "77e7d383beb96054547729f49c372b3d01e196ff" "Rev": "4300f6b0c8a7f09e521dd0af2cee27e28846e037"
}, },
{ {
"ImportPath": "google.golang.org/cloud/compute/metadata", "ImportPath": "google.golang.org/cloud/compute/metadata",
......
{ {
"kind": "discovery#restDescription", "kind": "discovery#restDescription",
"etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/avMl03W4ktR_Q7PS4O3ogtyT8Dc\"", "etag": "\"bRFOOrZKfO9LweMbPqu0kcu6De8/A2G_NAa29vne9MPSojupRQ5bVuo\"",
"discoveryVersion": "v1", "discoveryVersion": "v1",
"id": "cloudmonitoring:v2beta2", "id": "cloudmonitoring:v2beta2",
"name": "cloudmonitoring", "name": "cloudmonitoring",
"canonicalName": "Cloud Monitoring", "canonicalName": "Cloud Monitoring",
"version": "v2beta2", "version": "v2beta2",
"revision": "20150713", "revision": "20160314",
"title": "Cloud Monitoring API", "title": "Cloud Monitoring API",
"description": "API for accessing Google Cloud and API monitoring data.", "description": "Accesses Google Cloud Monitoring data.",
"ownerDomain": "google.com", "ownerDomain": "google.com",
"ownerName": "Google", "ownerName": "Google",
"icons": { "icons": {
......
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"math/rand"
"time"
)
type BackoffStrategy interface {
// Pause returns the duration of the next pause and true if the operation should be
// retried, or false if no further retries should be attempted.
Pause() (time.Duration, bool)
// Reset restores the strategy to its initial state.
Reset()
}
// ExponentialBackoff performs exponential backoff as per https://en.wikipedia.org/wiki/Exponential_backoff.
// The initial pause time is given by Base.
// Once the total pause time exceeds Max, Pause will indicate no further retries.
type ExponentialBackoff struct {
Base time.Duration
Max time.Duration
total time.Duration
n uint
}
func (eb *ExponentialBackoff) Pause() (time.Duration, bool) {
if eb.total > eb.Max {
return 0, false
}
// The next pause is selected from randomly from [0, 2^n * Base).
d := time.Duration(rand.Int63n((1 << eb.n) * int64(eb.Base)))
eb.total += d
eb.n++
return d, true
}
func (eb *ExponentialBackoff) Reset() {
eb.n = 0
eb.total = 0
}
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"bytes"
"io"
"google.golang.org/api/googleapi"
)
// MediaBuffer buffers data from an io.Reader to support uploading media in retryable chunks.
type MediaBuffer struct {
media io.Reader
chunk []byte // The current chunk which is pending upload. The capacity is the chunk size.
err error // Any error generated when populating chunk by reading media.
// The absolute position of chunk in the underlying media.
off int64
}
func NewMediaBuffer(media io.Reader, chunkSize int) *MediaBuffer {
return &MediaBuffer{media: media, chunk: make([]byte, 0, chunkSize)}
}
// Chunk returns the current buffered chunk, the offset in the underlying media
// from which the chunk is drawn, and the size of the chunk.
// Successive calls to Chunk return the same chunk between calls to Next.
func (mb *MediaBuffer) Chunk() (chunk io.Reader, off int64, size int, err error) {
// There may already be data in chunk if Next has not been called since the previous call to Chunk.
if mb.err == nil && len(mb.chunk) == 0 {
mb.err = mb.loadChunk()
}
return bytes.NewReader(mb.chunk), mb.off, len(mb.chunk), mb.err
}
// loadChunk will read from media into chunk, up to the capacity of chunk.
func (mb *MediaBuffer) loadChunk() error {
bufSize := cap(mb.chunk)
mb.chunk = mb.chunk[:bufSize]
read := 0
var err error
for err == nil && read < bufSize {
var n int
n, err = mb.media.Read(mb.chunk[read:])
read += n
}
mb.chunk = mb.chunk[:read]
return err
}
// Next advances to the next chunk, which will be returned by the next call to Chunk.
// Calls to Next without a corresponding prior call to Chunk will have no effect.
func (mb *MediaBuffer) Next() {
mb.off += int64(len(mb.chunk))
mb.chunk = mb.chunk[0:0]
}
type readerTyper struct {
io.Reader
googleapi.ContentTyper
}
// ReaderAtToReader adapts a ReaderAt to be used as a Reader.
// If ra implements googleapi.ContentTyper, then the returned reader
// will also implement googleapi.ContentTyper, delegating to ra.
func ReaderAtToReader(ra io.ReaderAt, size int64) io.Reader {
r := io.NewSectionReader(ra, 0, size)
if typer, ok := ra.(googleapi.ContentTyper); ok {
return readerTyper{r, typer}
}
return r
}
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
package gensupport package gensupport
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -18,12 +17,12 @@ import ( ...@@ -18,12 +17,12 @@ import (
const sniffBuffSize = 512 const sniffBuffSize = 512
func NewContentSniffer(r io.Reader) *ContentSniffer { func newContentSniffer(r io.Reader) *contentSniffer {
return &ContentSniffer{r: r} return &contentSniffer{r: r}
} }
// ContentSniffer wraps a Reader, and reports the content type determined by sniffing up to 512 bytes from the Reader. // contentSniffer wraps a Reader, and reports the content type determined by sniffing up to 512 bytes from the Reader.
type ContentSniffer struct { type contentSniffer struct {
r io.Reader r io.Reader
start []byte // buffer for the sniffed bytes. start []byte // buffer for the sniffed bytes.
err error // set to any error encountered while reading bytes to be sniffed. err error // set to any error encountered while reading bytes to be sniffed.
...@@ -32,133 +31,169 @@ type ContentSniffer struct { ...@@ -32,133 +31,169 @@ type ContentSniffer struct {
sniffed bool // set to true on first sniff. sniffed bool // set to true on first sniff.
} }
func (sct *ContentSniffer) Read(p []byte) (n int, err error) { func (cs *contentSniffer) Read(p []byte) (n int, err error) {
// Ensure that the content type is sniffed before any data is consumed from Reader. // Ensure that the content type is sniffed before any data is consumed from Reader.
_, _ = sct.ContentType() _, _ = cs.ContentType()
if len(sct.start) > 0 { if len(cs.start) > 0 {
n := copy(p, sct.start) n := copy(p, cs.start)
sct.start = sct.start[n:] cs.start = cs.start[n:]
return n, nil return n, nil
} }
// We may have read some bytes into start while sniffing, even if the read ended in an error. // We may have read some bytes into start while sniffing, even if the read ended in an error.
// We should first return those bytes, then the error. // We should first return those bytes, then the error.
if sct.err != nil { if cs.err != nil {
return 0, sct.err return 0, cs.err
} }
// Now we have handled all bytes that were buffered while sniffing. Now just delegate to the underlying reader. // Now we have handled all bytes that were buffered while sniffing. Now just delegate to the underlying reader.
return sct.r.Read(p) return cs.r.Read(p)
} }
// ContentType returns the sniffed content type, and whether the content type was succesfully sniffed. // ContentType returns the sniffed content type, and whether the content type was succesfully sniffed.
func (sct *ContentSniffer) ContentType() (string, bool) { func (cs *contentSniffer) ContentType() (string, bool) {
if sct.sniffed { if cs.sniffed {
return sct.ctype, sct.ctype != "" return cs.ctype, cs.ctype != ""
} }
sct.sniffed = true cs.sniffed = true
// If ReadAll hits EOF, it returns err==nil. // If ReadAll hits EOF, it returns err==nil.
sct.start, sct.err = ioutil.ReadAll(io.LimitReader(sct.r, sniffBuffSize)) cs.start, cs.err = ioutil.ReadAll(io.LimitReader(cs.r, sniffBuffSize))
// Don't try to detect the content type based on possibly incomplete data. // Don't try to detect the content type based on possibly incomplete data.
if sct.err != nil { if cs.err != nil {
return "", false return "", false
} }
sct.ctype = http.DetectContentType(sct.start) cs.ctype = http.DetectContentType(cs.start)
return sct.ctype, true return cs.ctype, true
} }
// IncludeMedia combines an existing HTTP body with media content to create a multipart/related HTTP body. // DetermineContentType determines the content type of the supplied reader.
// // If the content type is already known, it can be specified via ctype.
// bodyp is an in/out parameter. It should initially point to the // Otherwise, the content of media will be sniffed to determine the content type.
// reader of the application/json (or whatever) payload to send in the // If media implements googleapi.ContentTyper (deprecated), this will be used
// API request. It's updated to point to the multipart body reader. // instead of sniffing the content.
// // After calling DetectContentType the caller must not perform further reads on
// ctypep is an in/out parameter. It should initially point to the // media, but rather read from the Reader that is returned.
// content type of the bodyp, usually "application/json". It's updated func DetermineContentType(media io.Reader, ctype string) (io.Reader, string) {
// to the "multipart/related" content type, with random boundary. // Note: callers could avoid calling DetectContentType if ctype != "",
// // but doing the check inside this function reduces the amount of
// The return value is a function that can be used to close the bodyp Reader with an error. // generated code.
func IncludeMedia(media io.Reader, bodyp *io.Reader, ctypep *string) func() { if ctype != "" {
var mediaType string return media, ctype
media, mediaType = getMediaType(media) }
body, bodyType := *bodyp, *ctypep // For backwards compatability, allow clients to set content
// type by providing a ContentTyper for media.
if typer, ok := media.(googleapi.ContentTyper); ok {
return media, typer.ContentType()
}
pr, pw := io.Pipe() sniffer := newContentSniffer(media)
if ctype, ok := sniffer.ContentType(); ok {
return sniffer, ctype
}
// If content type could not be sniffed, reads from sniffer will eventually fail with an error.
return sniffer, ""
}
type typeReader struct {
io.Reader
typ string
}
// multipartReader combines the contents of multiple readers to creat a multipart/related HTTP body.
// Close must be called if reads from the multipartReader are abandoned before reaching EOF.
type multipartReader struct {
pr *io.PipeReader
pipeOpen bool
ctype string
}
func newMultipartReader(parts []typeReader) *multipartReader {
mp := &multipartReader{pipeOpen: true}
var pw *io.PipeWriter
mp.pr, pw = io.Pipe()
mpw := multipart.NewWriter(pw) mpw := multipart.NewWriter(pw)
*bodyp = pr mp.ctype = "multipart/related; boundary=" + mpw.Boundary()
*ctypep = "multipart/related; boundary=" + mpw.Boundary()
go func() { go func() {
w, err := mpw.CreatePart(typeHeader(bodyType)) for _, part := range parts {
if err != nil { w, err := mpw.CreatePart(typeHeader(part.typ))
mpw.Close() if err != nil {
pw.CloseWithError(fmt.Errorf("googleapi: body CreatePart failed: %v", err)) mpw.Close()
return pw.CloseWithError(fmt.Errorf("googleapi: CreatePart failed: %v", err))
} return
_, err = io.Copy(w, body) }
if err != nil { _, err = io.Copy(w, part.Reader)
mpw.Close() if err != nil {
pw.CloseWithError(fmt.Errorf("googleapi: body Copy failed: %v", err)) mpw.Close()
return pw.CloseWithError(fmt.Errorf("googleapi: Copy failed: %v", err))
return
}
} }
w, err = mpw.CreatePart(typeHeader(mediaType))
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: media CreatePart failed: %v", err))
return
}
_, err = io.Copy(w, media)
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: media Copy failed: %v", err))
return
}
mpw.Close() mpw.Close()
pw.Close() pw.Close()
}() }()
return func() { pw.CloseWithError(errAborted) } return mp
} }
var errAborted = errors.New("googleapi: upload aborted") func (mp *multipartReader) Read(data []byte) (n int, err error) {
return mp.pr.Read(data)
}
func getMediaType(media io.Reader) (io.Reader, string) { func (mp *multipartReader) Close() error {
if typer, ok := media.(googleapi.ContentTyper); ok { if !mp.pipeOpen {
return media, typer.ContentType() return nil
} }
mp.pipeOpen = false
return mp.pr.Close()
}
sniffer := NewContentSniffer(media) // CombineBodyMedia combines a json body with media content to create a multipart/related HTTP body.
typ, ok := sniffer.ContentType() // It returns a ReadCloser containing the combined body, and the overall "multipart/related" content type, with random boundary.
if !ok { //
// TODO(mcgreevy): Remove this default. It maintains the semantics of the existing code, // The caller must call Close on the returned ReadCloser if reads are abandoned before reaching EOF.
// but should not be relied on. func CombineBodyMedia(body io.Reader, bodyContentType string, media io.Reader, mediaContentType string) (io.ReadCloser, string) {
typ = "application/octet-stream" mp := newMultipartReader([]typeReader{
{body, bodyContentType},
{media, mediaContentType},
})
return mp, mp.ctype
}
func typeHeader(contentType string) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
if contentType != "" {
h.Set("Content-Type", contentType)
} }
return sniffer, typ return h
} }
// DetectMediaType detects and returns the content type of the provided media. // PrepareUpload determines whether the data in the supplied reader should be
// If the type can not be determined, "application/octet-stream" is returned. // uploaded in a single request, or in sequential chunks.
func DetectMediaType(media io.ReaderAt) string { // chunkSize is the size of the chunk that media should be split into.
if typer, ok := media.(googleapi.ContentTyper); ok { // If chunkSize is non-zero and the contents of media do not fit in a single
return typer.ContentType() // chunk (or there is an error reading media), then media will be returned as a
// MediaBuffer. Otherwise, media will be returned as a Reader.
//
// After PrepareUpload has been called, media should no longer be used: the
// media content should be accessed via one of the return values.
func PrepareUpload(media io.Reader, chunkSize int) (io.Reader, *MediaBuffer) {
if chunkSize == 0 { // do not chunk
return media, nil
} }
typ := "application/octet-stream" mb := NewMediaBuffer(media, chunkSize)
buf := make([]byte, 1024) rdr, _, _, err := mb.Chunk()
n, err := media.ReadAt(buf, 0)
buf = buf[:n] if err == io.EOF { // we can upload this in a single request
if err == nil || err == io.EOF { return rdr, nil
typ = http.DetectContentType(buf)
} }
return typ // err might be a non-EOF error. If it is, the next call to mb.Chunk will
} // return the same error. Returning a MediaBuffer ensures that this error
// will be handled at some point.
func typeHeader(contentType string) textproto.MIMEHeader { return nil, mb
h := make(textproto.MIMEHeader)
h.Set("Content-Type", contentType)
return h
} }
...@@ -4,12 +4,25 @@ ...@@ -4,12 +4,25 @@
package gensupport package gensupport
import "net/url" import (
"net/url"
"google.golang.org/api/googleapi"
)
// URLParams is a simplified replacement for url.Values // URLParams is a simplified replacement for url.Values
// that safely builds up URL parameters for encoding. // that safely builds up URL parameters for encoding.
type URLParams map[string][]string type URLParams map[string][]string
// Get returns the first value for the given key, or "".
func (u URLParams) Get(key string) string {
vs := u[key]
if len(vs) == 0 {
return ""
}
return vs[0]
}
// Set sets the key to value. // Set sets the key to value.
// It replaces any existing values. // It replaces any existing values.
func (u URLParams) Set(key, value string) { func (u URLParams) Set(key, value string) {
...@@ -29,3 +42,9 @@ func (u URLParams) SetMulti(key string, values []string) { ...@@ -29,3 +42,9 @@ func (u URLParams) SetMulti(key string, values []string) {
func (u URLParams) Encode() string { func (u URLParams) Encode() string {
return url.Values(u).Encode() return url.Values(u).Encode()
} }
func SetOptions(u URLParams, opts ...googleapi.CallOption) {
for _, o := range opts {
u.Set(o.Get())
}
}
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"fmt"
"io"
"net/http"
"sync"
"time"
"golang.org/x/net/context"
"golang.org/x/net/context/ctxhttp"
)
const (
// statusResumeIncomplete is the code returned by the Google uploader
// when the transfer is not yet complete.
statusResumeIncomplete = 308
// statusTooManyRequests is returned by the storage API if the
// per-project limits have been temporarily exceeded. The request
// should be retried.
// https://cloud.google.com/storage/docs/json_api/v1/status-codes#standardcodes
statusTooManyRequests = 429
)
// ResumableUpload is used by the generated APIs to provide resumable uploads.
// It is not used by developers directly.
type ResumableUpload struct {
Client *http.Client
// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
URI string
UserAgent string // User-Agent for header of the request
// Media is the object being uploaded.
Media *MediaBuffer
// MediaType defines the media type, e.g. "image/jpeg".
MediaType string
mu sync.Mutex // guards progress
progress int64 // number of bytes uploaded so far
// Callback is an optional function that will be periodically called with the cumulative number of bytes uploaded.
Callback func(int64)
// If not specified, a default exponential backoff strategy will be used.
Backoff BackoffStrategy
}
// Progress returns the number of bytes uploaded at this point.
func (rx *ResumableUpload) Progress() int64 {
rx.mu.Lock()
defer rx.mu.Unlock()
return rx.progress
}
// doUploadRequest performs a single HTTP request to upload data.
// off specifies the offset in rx.Media from which data is drawn.
// size is the number of bytes in data.
// final specifies whether data is the final chunk to be uploaded.
func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader, off, size int64, final bool) (*http.Response, error) {
req, err := http.NewRequest("POST", rx.URI, data)
if err != nil {
return nil, err
}
req.ContentLength = size
var contentRange string
if final {
if size == 0 {
contentRange = fmt.Sprintf("bytes */%v", off)
} else {
contentRange = fmt.Sprintf("bytes %v-%v/%v", off, off+size-1, off+size)
}
} else {
contentRange = fmt.Sprintf("bytes %v-%v/*", off, off+size-1)
}
req.Header.Set("Content-Range", contentRange)
req.Header.Set("Content-Type", rx.MediaType)
req.Header.Set("User-Agent", rx.UserAgent)
return ctxhttp.Do(ctx, rx.Client, req)
}
// reportProgress calls a user-supplied callback to report upload progress.
// If old==updated, the callback is not called.
func (rx *ResumableUpload) reportProgress(old, updated int64) {
if updated-old == 0 {
return
}
rx.mu.Lock()
rx.progress = updated
rx.mu.Unlock()
if rx.Callback != nil {
rx.Callback(updated)
}
}
// transferChunk performs a single HTTP request to upload a single chunk from rx.Media.
func (rx *ResumableUpload) transferChunk(ctx context.Context) (*http.Response, error) {
chunk, off, size, err := rx.Media.Chunk()
done := err == io.EOF
if !done && err != nil {
return nil, err
}
res, err := rx.doUploadRequest(ctx, chunk, off, int64(size), done)
if err != nil {
return res, err
}
if res.StatusCode == statusResumeIncomplete || res.StatusCode == http.StatusOK {
rx.reportProgress(off, off+int64(size))
}
if res.StatusCode == statusResumeIncomplete {
rx.Media.Next()
}
return res, nil
}
func contextDone(ctx context.Context) bool {
select {
case <-ctx.Done():
return true
default:
return false
}
}
// Upload starts the process of a resumable upload with a cancellable context.
// It retries using the provided back off strategy until cancelled or the
// strategy indicates to stop retrying.
// It is called from the auto-generated API code and is not visible to the user.
// rx is private to the auto-generated API code.
// Exactly one of resp or err will be nil. If resp is non-nil, the caller must call resp.Body.Close.
func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err error) {
var pause time.Duration
backoff := rx.Backoff
if backoff == nil {
backoff = DefaultBackoffStrategy()
}
for {
// Ensure that we return in the case of cancelled context, even if pause is 0.
if contextDone(ctx) {
return nil, ctx.Err()
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(pause):
}
resp, err = rx.transferChunk(ctx)
var status int
if resp != nil {
status = resp.StatusCode
}
// Check if we should retry the request.
if shouldRetry(status, err) {
var retry bool
pause, retry = backoff.Pause()
if retry {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
continue
}
}
// If the chunk was uploaded successfully, but there's still
// more to go, upload the next chunk without any delay.
if status == statusResumeIncomplete {
pause = 0
backoff.Reset()
resp.Body.Close()
continue
}
// It's possible for err and resp to both be non-nil here, but we expose a simpler
// contract to our callers: exactly one of resp and err will be non-nil. This means
// that any response body must be closed here before returning a non-nil error.
if err != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
return nil, err
}
return resp, nil
}
}
package gensupport
import (
"io"
"net"
"net/http"
"time"
"golang.org/x/net/context"
)
// Retry invokes the given function, retrying it multiple times if the connection failed or
// the HTTP status response indicates the request should be attempted again. ctx may be nil.
func Retry(ctx context.Context, f func() (*http.Response, error), backoff BackoffStrategy) (*http.Response, error) {
for {
resp, err := f()
var status int
if resp != nil {
status = resp.StatusCode
}
// Return if we shouldn't retry.
pause, retry := backoff.Pause()
if !shouldRetry(status, err) || !retry {
return resp, err
}
// Ensure the response body is closed, if any.
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
// Pause, but still listen to ctx.Done if context is not nil.
var done <-chan struct{}
if ctx != nil {
done = ctx.Done()
}
select {
case <-done:
return nil, ctx.Err()
case <-time.After(pause):
}
}
}
// DefaultBackoffStrategy returns a default strategy to use for retrying failed upload requests.
func DefaultBackoffStrategy() BackoffStrategy {
return &ExponentialBackoff{
Base: 250 * time.Millisecond,
Max: 16 * time.Second,
}
}
// shouldRetry returns true if the HTTP response / error indicates that the
// request should be attempted again.
func shouldRetry(status int, err error) bool {
// Retry for 5xx response codes.
if 500 <= status && status < 600 {
return true
}
// Retry on statusTooManyRequests{
if status == statusTooManyRequests {
return true
}
// Retry on unexpected EOFs and temporary network errors.
if err == io.ErrUnexpectedEOF {
return true
}
if err, ok := err.(net.Error); ok {
return err.Temporary()
}
return false
}
...@@ -14,14 +14,8 @@ import ( ...@@ -14,14 +14,8 @@ import (
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"regexp"
"strconv"
"strings" "strings"
"sync"
"time"
"golang.org/x/net/context"
"golang.org/x/net/context/ctxhttp"
"google.golang.org/api/googleapi/internal/uritemplates" "google.golang.org/api/googleapi/internal/uritemplates"
) )
...@@ -53,14 +47,15 @@ type ServerResponse struct { ...@@ -53,14 +47,15 @@ type ServerResponse struct {
const ( const (
Version = "0.5" Version = "0.5"
// statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete.
statusResumeIncomplete = 308
// UserAgent is the header string used to identify this package. // UserAgent is the header string used to identify this package.
UserAgent = "google-api-go-client/" + Version UserAgent = "google-api-go-client/" + Version
// uploadPause determines the delay between failed upload attempts // The default chunk size to use for resumable uplods if not specified by the user.
uploadPause = 1 * time.Second DefaultUploadChunkSize = 8 * 1024 * 1024
// The minimum chunk size that can be used for resumable uploads. All
// user-specified chunk sizes must be multiple of this value.
MinUploadChunkSize = 256 * 1024
) )
// Error contains an error response from the server. // Error contains an error response from the server.
...@@ -217,134 +212,60 @@ func (w countingWriter) Write(p []byte) (int, error) { ...@@ -217,134 +212,60 @@ func (w countingWriter) Write(p []byte) (int, error) {
// The remaining usable pieces of resumable uploads is exposed in each auto-generated API. // The remaining usable pieces of resumable uploads is exposed in each auto-generated API.
type ProgressUpdater func(current, total int64) type ProgressUpdater func(current, total int64)
// ResumableUpload is used by the generated APIs to provide resumable uploads. type MediaOption interface {
// It is not used by developers directly. setOptions(o *MediaOptions)
type ResumableUpload struct {
Client *http.Client
// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
URI string
UserAgent string // User-Agent for header of the request
// Media is the object being uploaded.
Media io.ReaderAt
// MediaType defines the media type, e.g. "image/jpeg".
MediaType string
// ContentLength is the full size of the object being uploaded.
ContentLength int64
mu sync.Mutex // guards progress
progress int64 // number of bytes uploaded so far
// Callback is an optional function that will be periodically called with the cumulative number of bytes uploaded.
Callback func(int64)
} }
var ( type contentTypeOption string
// rangeRE matches the transfer status response from the server. $1 is the last byte index uploaded.
rangeRE = regexp.MustCompile(`^bytes=0\-(\d+)$`)
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
chunkSize int64 = 1 << 18
)
// Progress returns the number of bytes uploaded at this point. func (ct contentTypeOption) setOptions(o *MediaOptions) {
func (rx *ResumableUpload) Progress() int64 { o.ContentType = string(ct)
rx.mu.Lock() if o.ContentType == "" {
defer rx.mu.Unlock() o.ForceEmptyContentType = true
return rx.progress }
} }
func (rx *ResumableUpload) transferStatus(ctx context.Context) (int64, *http.Response, error) { // ContentType returns a MediaOption which sets the Content-Type header for media uploads.
req, _ := http.NewRequest("POST", rx.URI, nil) // If ctype is empty, the Content-Type header will be omitted.
req.ContentLength = 0 func ContentType(ctype string) MediaOption {
req.Header.Set("User-Agent", rx.UserAgent) return contentTypeOption(ctype)
req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength)) }
res, err := ctxhttp.Do(ctx, rx.Client, req)
if err != nil || res.StatusCode != statusResumeIncomplete { type chunkSizeOption int
return 0, res, err
} func (cs chunkSizeOption) setOptions(o *MediaOptions) {
var start int64 size := int(cs)
if m := rangeRE.FindStringSubmatch(res.Header.Get("Range")); len(m) == 2 { if size%MinUploadChunkSize != 0 {
start, err = strconv.ParseInt(m[1], 10, 64) size += MinUploadChunkSize - (size % MinUploadChunkSize)
if err != nil {
return 0, nil, fmt.Errorf("unable to parse range size %v", m[1])
}
start += 1 // Start at the next byte
} }
return start, res, nil o.ChunkSize = size
} }
type chunk struct { // ChunkSize returns a MediaOption which sets the chunk size for media uploads.
body io.Reader // size will be rounded up to the nearest multiple of 256K.
size int64 // Media which contains fewer than size bytes will be uploaded in a single request.
err error // Media which contains size bytes or more will be uploaded in separate chunks.
// If size is zero, media will be uploaded in a single request.
func ChunkSize(size int) MediaOption {
return chunkSizeOption(size)
} }
func (rx *ResumableUpload) transferChunks(ctx context.Context) (*http.Response, error) { // MediaOptions stores options for customizing media upload. It is not used by developers directly.
start, res, err := rx.transferStatus(ctx) type MediaOptions struct {
if err != nil || res.StatusCode != statusResumeIncomplete { ContentType string
if err == context.Canceled { ForceEmptyContentType bool
return &http.Response{StatusCode: http.StatusRequestTimeout}, err
}
return res, err
}
for { ChunkSize int
select { // Check for cancellation
case <-ctx.Done():
res.StatusCode = http.StatusRequestTimeout
return res, ctx.Err()
default:
}
reqSize := rx.ContentLength - start
if reqSize > chunkSize {
reqSize = chunkSize
}
r := io.NewSectionReader(rx.Media, start, reqSize)
req, _ := http.NewRequest("POST", rx.URI, r)
req.ContentLength = reqSize
req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
req.Header.Set("Content-Type", rx.MediaType)
req.Header.Set("User-Agent", rx.UserAgent)
res, err = ctxhttp.Do(ctx, rx.Client, req)
start += reqSize
if err == nil && (res.StatusCode == statusResumeIncomplete || res.StatusCode == http.StatusOK) {
rx.mu.Lock()
rx.progress = start // keep track of number of bytes sent so far
rx.mu.Unlock()
if rx.Callback != nil {
rx.Callback(start)
}
}
if err != nil || res.StatusCode != statusResumeIncomplete {
break
}
}
return res, err
} }
var sleep = time.Sleep // override in unit tests // ProcessMediaOptions stores options from opts in a MediaOptions.
// It is not used by developers directly.
// Upload starts the process of a resumable upload with a cancellable context. func ProcessMediaOptions(opts []MediaOption) *MediaOptions {
// It retries indefinitely (with a pause of uploadPause between attempts) until cancelled. mo := &MediaOptions{ChunkSize: DefaultUploadChunkSize}
// It is called from the auto-generated API code and is not visible to the user. for _, o := range opts {
// rx is private to the auto-generated API code. o.setOptions(mo)
func (rx *ResumableUpload) Upload(ctx context.Context) (*http.Response, error) {
var res *http.Response
var err error
for {
res, err = rx.transferChunks(ctx)
if err != nil || res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
return res, err
}
select { // Check for cancellation
case <-ctx.Done():
res.StatusCode = http.StatusRequestTimeout
return res, ctx.Err()
default:
}
sleep(uploadPause)
} }
return res, err return mo
} }
func ResolveRelative(basestr, relstr string) string { func ResolveRelative(basestr, relstr string) string {
...@@ -471,3 +392,41 @@ func CombineFields(s []Field) string { ...@@ -471,3 +392,41 @@ func CombineFields(s []Field) string {
} }
return strings.Join(r, ",") return strings.Join(r, ",")
} }
// A CallOption is an optional argument to an API call.
// It should be treated as an opaque value by users of Google APIs.
//
// A CallOption is something that configures an API call in a way that is
// not specific to that API; for instance, controlling the quota user for
// an API call is common across many APIs, and is thus a CallOption.
type CallOption interface {
Get() (key, value string)
}
// QuotaUser returns a CallOption that will set the quota user for a call.
// The quota user can be used by server-side applications to control accounting.
// It can be an arbitrary string up to 40 characters, and will override UserIP
// if both are provided.
func QuotaUser(u string) CallOption { return quotaUser(u) }
type quotaUser string
func (q quotaUser) Get() (string, string) { return "quotaUser", string(q) }
// UserIP returns a CallOption that will set the "userIp" parameter of a call.
// This should be the IP address of the originating request.
func UserIP(ip string) CallOption { return userIP(ip) }
type userIP string
func (i userIP) Get() (string, string) { return "userIp", string(i) }
// Trace returns a CallOption that enables diagnostic tracing for a call.
// traceToken is an ID supplied by Google support.
func Trace(traceToken string) CallOption { return traceTok(traceToken) }
type traceTok string
func (t traceTok) Get() (string, string) { return "trace", "token:" + string(t) }
// TODO: Fields too
...@@ -2,26 +2,15 @@ ...@@ -2,26 +2,15 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// Package uritemplates is a level 4 implementation of RFC 6570 (URI // Package uritemplates is a level 3 implementation of RFC 6570 (URI
// Template, http://tools.ietf.org/html/rfc6570). // Template, http://tools.ietf.org/html/rfc6570).
// // uritemplates does not support composite values (in Go: slices or maps)
// To use uritemplates, parse a template string and expand it with a value // and so does not qualify as a level 4 implementation.
// map:
//
// template, _ := uritemplates.Parse("https://api.github.com/repos{/user,repo}")
// values := make(map[string]interface{})
// values["user"] = "jtacoma"
// values["repo"] = "uritemplates"
// expanded, _ := template.ExpandString(values)
// fmt.Printf(expanded)
//
package uritemplates package uritemplates
import ( import (
"bytes" "bytes"
"errors" "errors"
"fmt"
"reflect"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
...@@ -45,52 +34,47 @@ func pctEncode(src []byte) []byte { ...@@ -45,52 +34,47 @@ func pctEncode(src []byte) []byte {
return dst return dst
} }
func escape(s string, allowReserved bool) (escaped string) { func escape(s string, allowReserved bool) string {
if allowReserved { if allowReserved {
escaped = string(reserved.ReplaceAllFunc([]byte(s), pctEncode)) return string(reserved.ReplaceAllFunc([]byte(s), pctEncode))
} else {
escaped = string(unreserved.ReplaceAllFunc([]byte(s), pctEncode))
} }
return escaped return string(unreserved.ReplaceAllFunc([]byte(s), pctEncode))
} }
// A UriTemplate is a parsed representation of a URI template. // A uriTemplate is a parsed representation of a URI template.
type UriTemplate struct { type uriTemplate struct {
raw string raw string
parts []templatePart parts []templatePart
} }
// Parse parses a URI template string into a UriTemplate object. // parse parses a URI template string into a uriTemplate object.
func Parse(rawtemplate string) (template *UriTemplate, err error) { func parse(rawTemplate string) (*uriTemplate, error) {
template = new(UriTemplate) split := strings.Split(rawTemplate, "{")
template.raw = rawtemplate parts := make([]templatePart, len(split)*2-1)
split := strings.Split(rawtemplate, "{")
template.parts = make([]templatePart, len(split)*2-1)
for i, s := range split { for i, s := range split {
if i == 0 { if i == 0 {
if strings.Contains(s, "}") { if strings.Contains(s, "}") {
err = errors.New("unexpected }") return nil, errors.New("unexpected }")
break
}
template.parts[i].raw = s
} else {
subsplit := strings.Split(s, "}")
if len(subsplit) != 2 {
err = errors.New("malformed template")
break
} }
expression := subsplit[0] parts[i].raw = s
template.parts[i*2-1], err = parseExpression(expression) continue
if err != nil {
break
}
template.parts[i*2].raw = subsplit[1]
} }
subsplit := strings.Split(s, "}")
if len(subsplit) != 2 {
return nil, errors.New("malformed template")
}
expression := subsplit[0]
var err error
parts[i*2-1], err = parseExpression(expression)
if err != nil {
return nil, err
}
parts[i*2].raw = subsplit[1]
} }
if err != nil { return &uriTemplate{
template = nil raw: rawTemplate,
} parts: parts,
return template, err }, nil
} }
type templatePart struct { type templatePart struct {
...@@ -160,6 +144,8 @@ func parseExpression(expression string) (result templatePart, err error) { ...@@ -160,6 +144,8 @@ func parseExpression(expression string) (result templatePart, err error) {
} }
func parseTerm(term string) (result templateTerm, err error) { func parseTerm(term string) (result templateTerm, err error) {
// TODO(djd): Remove "*" suffix parsing once we check that no APIs have
// mistakenly used that attribute.
if strings.HasSuffix(term, "*") { if strings.HasSuffix(term, "*") {
result.explode = true result.explode = true
term = term[:len(term)-1] term = term[:len(term)-1]
...@@ -185,175 +171,50 @@ func parseTerm(term string) (result templateTerm, err error) { ...@@ -185,175 +171,50 @@ func parseTerm(term string) (result templateTerm, err error) {
} }
// Expand expands a URI template with a set of values to produce a string. // Expand expands a URI template with a set of values to produce a string.
func (self *UriTemplate) Expand(value interface{}) (string, error) { func (t *uriTemplate) Expand(values map[string]string) string {
values, ismap := value.(map[string]interface{})
if !ismap {
if m, ismap := struct2map(value); !ismap {
return "", errors.New("expected map[string]interface{}, struct, or pointer to struct.")
} else {
return self.Expand(m)
}
}
var buf bytes.Buffer var buf bytes.Buffer
for _, p := range self.parts { for _, p := range t.parts {
err := p.expand(&buf, values) p.expand(&buf, values)
if err != nil {
return "", err
}
} }
return buf.String(), nil return buf.String()
} }
func (self *templatePart) expand(buf *bytes.Buffer, values map[string]interface{}) error { func (tp *templatePart) expand(buf *bytes.Buffer, values map[string]string) {
if len(self.raw) > 0 { if len(tp.raw) > 0 {
buf.WriteString(self.raw) buf.WriteString(tp.raw)
return nil return
} }
var zeroLen = buf.Len() var first = true
buf.WriteString(self.first) for _, term := range tp.terms {
var firstLen = buf.Len()
for _, term := range self.terms {
value, exists := values[term.name] value, exists := values[term.name]
if !exists { if !exists {
continue continue
} }
if buf.Len() != firstLen { if first {
buf.WriteString(self.sep) buf.WriteString(tp.first)
} first = false
switch v := value.(type) { } else {
case string: buf.WriteString(tp.sep)
self.expandString(buf, term, v)
case []interface{}:
self.expandArray(buf, term, v)
case map[string]interface{}:
if term.truncate > 0 {
return errors.New("cannot truncate a map expansion")
}
self.expandMap(buf, term, v)
default:
if m, ismap := struct2map(value); ismap {
if term.truncate > 0 {
return errors.New("cannot truncate a map expansion")
}
self.expandMap(buf, term, m)
} else {
str := fmt.Sprintf("%v", value)
self.expandString(buf, term, str)
}
} }
tp.expandString(buf, term, value)
} }
if buf.Len() == firstLen {
original := buf.Bytes()[:zeroLen]
buf.Reset()
buf.Write(original)
}
return nil
} }
func (self *templatePart) expandName(buf *bytes.Buffer, name string, empty bool) { func (tp *templatePart) expandName(buf *bytes.Buffer, name string, empty bool) {
if self.named { if tp.named {
buf.WriteString(name) buf.WriteString(name)
if empty { if empty {
buf.WriteString(self.ifemp) buf.WriteString(tp.ifemp)
} else { } else {
buf.WriteString("=") buf.WriteString("=")
} }
} }
} }
func (self *templatePart) expandString(buf *bytes.Buffer, t templateTerm, s string) { func (tp *templatePart) expandString(buf *bytes.Buffer, t templateTerm, s string) {
if len(s) > t.truncate && t.truncate > 0 { if len(s) > t.truncate && t.truncate > 0 {
s = s[:t.truncate] s = s[:t.truncate]
} }
self.expandName(buf, t.name, len(s) == 0) tp.expandName(buf, t.name, len(s) == 0)
buf.WriteString(escape(s, self.allowReserved)) buf.WriteString(escape(s, tp.allowReserved))
}
func (self *templatePart) expandArray(buf *bytes.Buffer, t templateTerm, a []interface{}) {
if len(a) == 0 {
return
} else if !t.explode {
self.expandName(buf, t.name, false)
}
for i, value := range a {
if t.explode && i > 0 {
buf.WriteString(self.sep)
} else if i > 0 {
buf.WriteString(",")
}
var s string
switch v := value.(type) {
case string:
s = v
default:
s = fmt.Sprintf("%v", v)
}
if len(s) > t.truncate && t.truncate > 0 {
s = s[:t.truncate]
}
if self.named && t.explode {
self.expandName(buf, t.name, len(s) == 0)
}
buf.WriteString(escape(s, self.allowReserved))
}
}
func (self *templatePart) expandMap(buf *bytes.Buffer, t templateTerm, m map[string]interface{}) {
if len(m) == 0 {
return
}
if !t.explode {
self.expandName(buf, t.name, len(m) == 0)
}
var firstLen = buf.Len()
for k, value := range m {
if firstLen != buf.Len() {
if t.explode {
buf.WriteString(self.sep)
} else {
buf.WriteString(",")
}
}
var s string
switch v := value.(type) {
case string:
s = v
default:
s = fmt.Sprintf("%v", v)
}
if t.explode {
buf.WriteString(escape(k, self.allowReserved))
buf.WriteRune('=')
buf.WriteString(escape(s, self.allowReserved))
} else {
buf.WriteString(escape(k, self.allowReserved))
buf.WriteRune(',')
buf.WriteString(escape(s, self.allowReserved))
}
}
}
func struct2map(v interface{}) (map[string]interface{}, bool) {
value := reflect.ValueOf(v)
switch value.Type().Kind() {
case reflect.Ptr:
return struct2map(value.Elem().Interface())
case reflect.Struct:
m := make(map[string]interface{})
for i := 0; i < value.NumField(); i++ {
tag := value.Type().Field(i).Tag
var name string
if strings.Contains(string(tag), ":") {
name = tag.Get("uri")
} else {
name = strings.TrimSpace(string(tag))
}
if len(name) == 0 {
name = value.Type().Field(i).Name
}
m[name] = value.Field(i).Interface()
}
return m, true
}
return nil, false
} }
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uritemplates package uritemplates
func Expand(path string, expansions map[string]string) (string, error) { func Expand(path string, values map[string]string) (string, error) {
template, err := Parse(path) template, err := parse(path)
if err != nil { if err != nil {
return "", err return "", err
} }
values := make(map[string]interface{}) return template.Expand(values), nil
for k, v := range expansions {
values[k] = v
}
return template.Expand(values)
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment