New Godeps LICENSE generation tool.

Godeps/LICENSES.md

-Dependency Licenses
--------------------
-
-Package | License
-------- | -------
-bitbucket.org/bertimus9/systemstat | MITname
-bitbucket.org/ww/goautoneg | spdxBSD3
-github.com/abbot/go-http-auth | Apache-2
-github.com/appc/cni | Apache-2
-github.com/appc/spec | Apache-2
-github.com/armon/go-metrics | MITname
-github.com/aws/aws-sdk-go | Apache-2
-github.com/beorn7/perks/quantile | MIT?
-github.com/blang/semver | MITname
-github.com/boltdb/bolt | MITname
-github.com/camlistore/go4 | Apache-2
-github.com/ClusterHQ/flocker-go | Apache-2
-github.com/codegangsta/negroni | MITname
-github.com/coreos/etcd | Apache-2
-github.com/coreos/go-etcd | Apache-2
-github.com/coreos/go-oidc | Apache-2
-github.com/coreos/go-semver | Apache-2
-github.com/coreos/go-systemd | Apache-2
-github.com/coreos/pkg | Apache-2
-github.com/coreos/rkt | MITname
-github.com/cpuguy83/go-md2man | MITname
-github.com/davecgh/go-spew | MIToldwithoutSellandNoDocumentationRequi
-github.com/daviddengcn/go-colortext | BSD?
-github.com/dgrijalva/jwt-go | spdxMIT
-github.com/docker/docker | Apache-2
-github.com/docker/docker/pkg/symlink | spdxBSD3
-github.com/docker/go-units | Apache-2
-github.com/docker/spdystream | SeeFile
-github.com/elazarl/go-bindata-assetfs | spdxBSD2
-github.com/elazarl/goproxy | BSDWarr
-github.com/emicklei/go-restful | MITname
-github.com/evanphx/json-patch | BSDWarr
-github.com/fsouza/go-dockerclient | spdxBSD2
-github.com/garyburd/redigo/internal | ApachesPermLim
-github.com/garyburd/redigo/redis | ApachesPermLim
-github.com/ghodss/yaml | MITname
-github.com/go-ini/ini | Apache-2
-github.com/godbus/dbus | spdxBSD2
-github.com/gogo/protobuf | spdxBSD3
-github.com/golang/glog | Apache-2
-github.com/golang/groupcache | Apache-2
-github.com/golang/protobuf | spdxBSD3
-github.com/google/btree | Apache-2
-github.com/google/cadvisor | Apache-2
-github.com/google/gofuzz | Apache-2
-github.com/gorilla/context | spdxBSD3
-github.com/gorilla/mux | spdxBSD3
-github.com/hashicorp/go-msgpack | spdxBSD3
-github.com/hashicorp/golang-lru | IntelPart08
-github.com/hashicorp/raft | IntelPart08
-github.com/hashicorp/raft-boltdb | IntelPart08
-github.com/hawkular/hawkular-client-go | Apache-2
-github.com/imdario/mergo | spdxBSD3
-github.com/inconshreveable/mousetrap | Apache-2
-github.com/influxdb/influxdb | MITname
-github.com/jmespath/go-jmespath | Apache-2
-github.com/jonboulle/clockwork | Apache-2
-github.com/juju/ratelimit | LesserExceptionGPLVer3-TOOLONG
-github.com/kardianos/osext | spdxBSD3
-github.com/kr/pty | spdxMIT
-github.com/matttproud/golang_protobuf_extensions | Apache-2
-github.com/mesos/mesos-go | Apache-2
-github.com/miekg/dns | spdxBSD3
-github.com/mistifyio/go-zfs | Apache-2
-github.com/mitchellh/mapstructure | MITname
-github.com/mvdan/xurls | spdxBSD3
-github.com/mxk/go-flowrate | spdxBSD3
-github.com/onsi/ginkgo | spdxMIT
-github.com/onsi/gomega | spdxMIT
-github.com/opencontainers/runc | Apache-2
-github.com/pborman/uuid | spdxBSD3
-github.com/pmezard/go-difflib | BSD3
-github.com/prometheus/client_golang | Apache-2
-github.com/prometheus/client_model | Apache-2
-github.com/prometheus/common/expfmt | Apache-2
-github.com/prometheus/common/model | Apache-2
-github.com/prometheus/procfs | Apache-2
-github.com/rackspace/gophercloud | Apache-2
-github.com/russross/blackfriday | AsIsVariant2-TOOLONG
-github.com/samuel/go-zookeeper | spdxBSD3
-github.com/scalingdata/gcfg | spdxBSD2
-github.com/seccomp/libseccomp-golang | AllRights-TOOLONG
-github.com/shurcooL/sanitized_anchor_name | MIT?
-github.com/Sirupsen/logrus | MITname
-github.com/skynetservices/skydns | MITname
-github.com/spf13/cobra | Apache-2
-github.com/spf13/pflag | spdxBSD3
-github.com/stretchr/objx | MIT?
-github.com/stretchr/testify | spdxMIT
-github.com/syndtr/gocapability | spdxBSD2
-github.com/ugorji/go | MITname
-github.com/vishvananda/netlink | Apache-2
-github.com/xiang90/probing | MITname
-github.com/xyproto/simpleredis | MITname
-golang.org/x/crypto | spdxBSD3
-golang.org/x/exp | spdxBSD3
-golang.org/x/net | spdxBSD3
-golang.org/x/oauth2 | spdxBSD3
-golang.org/x/sys | spdxBSD3
-golang.org/x/tools | spdxBSD3
-google.golang.org/api | spdxBSD3
-google.golang.org/cloud | Apache-2
-google.golang.org/grpc | spdxBSD3
-gopkg.in/natefinch/lumberjack.v2 | MITname
-gopkg.in/yaml.v2 | LesserExceptionGPLVer3-TOOLONG
-k8s.io/heapster | Apache-2
-speter.net/go/exp/math/dec/inf | spdxBSD2

build/common.sh

@@ -1030,7 +1030,7 @@ function kube::release::package_full_tarball() {
   cp -R "${KUBE_ROOT}/examples" "${release_stage}/"
   cp -R "${KUBE_ROOT}/docs" "${release_stage}/"
   cp "${KUBE_ROOT}/README.md" "${release_stage}/"
-  cp "${KUBE_ROOT}/LICENSE" "${release_stage}/"
+  cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
   cp "${KUBE_ROOT}/Vagrantfile" "${release_stage}/"
   mkdir -p "${release_stage}/contrib/completions/bash"
   cp "${KUBE_ROOT}/contrib/completions/bash/kubectl" "${release_stage}/contrib/completions/bash"

docs/devel/development.md

@@ -228,9 +228,8 @@ It is sometimes expedient to manually fix the /Godeps/godeps.json file to minimi
 
 Please send dependency updates in separate commits within your PR, for easier reviewing.
 
-6) If you updated the Godeps, please also update `Godeps/LICENSES.md` by running `hack/update-godep-licenses.sh`.
+6) If you updated the Godeps, please also update `Godeps/LICENSES` by running `hack/update-godep-licenses.sh`.
 
-_If Godep does not automatically vendor the proper license file for a new dependency, be sure to add an exception entry to `hack/update-godep-licenses.sh`._
 
 ## Unit tests
 

hack/update-godep-licenses.sh

 #!/bin/bash
-
 # Copyright 2015 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,30 +15,196 @@
 
 # Update the Godeps/LICENSES.md document.
 # Generates a table of Godep dependencies and their license.
-# Requires:
-#    docker
-#    mesosphere/godep-licenses (docker image) - source: https://github.com/mesosphere/godep-licenses
+#
 # Usage:
-#    Run every time a license file is added/modified within /Godeps to update /Godeps/LICENSES.md.
-#    Add exceptions (-e <repo>:<license>) for any dependency (project) vendored by Godep
-#      that has a known license that isn't vendored by Godep or can't be found by godep-licenses.
+#    $0 [--create-missing] [/path/to/licenses]
+#
+#    --create-missing will write the files that only exist upstream, locally.
+#    This option is mostly used for testing as we cannot check-in any of the
+#    additionally created files into the godep auto-generated tree.
+#
+#    Run every time a license file is added/modified within /Godeps to
+#    update /Godeps/LICENSES
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
-KUBE_ROOT="${KUBE_ROOT:-$(cd "$(dirname "${BASH_SOURCE}")/.." && pwd -P)}"
+###############################################################################
+# Manage the state of LICENSE/COPYRIGHT files
+# Default operation is to check to see if a file is in the state file.
+#
+# @optparam -a    Add the file to the state file
+# @param    file  The file to check or add
+# @return 1 when no file is found in state file
+#
+file_state () {
+  local add=0
+  case "$1" in
+    -a) add=1;shift ;;
+  esac
+  local file=$1
+
+  # If we're ignoring state, then return 1
+  ((CREATE_MISSING)) && return 1
+
+  # initialize if step 1
+  if ((add)); then
+    echo "${file}" >> ${GODEPS_STATE}
+    return 0
+  fi
+
+  # Get return code from grep itself
+  # Redirect stderr so that a missing state file returns 1 quietly
+  LANG=C egrep -wq "^${file}$" ${GODEPS_STATE} 2>/dev/null
+}
+
+###############################################################################
+# Process package content
+#
+# @param package  The incoming package name
+# @param type     The type of content (LICENSE or COPYRIGHT)
+#
+process_content () {
+  local package=$1
+  local type=$2
+  local f
+  local package_root
+  local ensure_pattern
+  local package_root_url
+  local dir_root
+  local find_maxdepth
+  local find_names
+  local -a local_files=()
+  local -a remote_files=()
+
+  # Necessary to expand {}
+  case ${type} in
+      LICENSE) remote_files=(LICENSE{,.code,.txt,.md})
+               find_names=(-iname 'licen[sc]e*')
+               find_maxdepth=1
+               # Sadly inconsistent in the wild, but mostly license files
+               # containing copyrights, but no readme/notice files containing
+               # licenses (except to "see license file")
+               ensure_pattern="License|Copyright"
+               ;;
+    # We search readmes for copyrights and this includes notice files as well
+    # Look in as many places as we find files matching
+    COPYRIGHT) remote_files=(NOTICE{,.txt} README{,.md})
+               find_names=(-iname 'notice*' -o -iname 'readme*')
+               find_maxdepth=3
+               ensure_pattern="Copyright"
+               ;;
+  esac
+
+  # Start search at package root
+  case ${package} in
+    github.com/*|golang.org/*|bitbucket.org/*)
+     package_root=$(echo ${package} |awk -F/ '{print $1"/"$2"/"$3 }')
+     ;;
+               *) 
+     package_root=$(echo ${package} |awk -F/ '{print $1"/"$2 }')
+     ;;
+  esac
+  # if github.com, rewrite package root url, otherwise take as is
+  package_root_url="${package_root/github.com/raw.githubusercontent.com}"
 
+  # Find LOCAL files first - only root and package level
+  for dir_root in ${package} ${package_root}; do
+    # One (set) of these is fine
+    local_files+=($(find ${GODEPS_SRC}/${dir_root} -xdev -follow \
+                        -maxdepth ${find_maxdepth} -type f "${find_names[@]}"))
+  done
+  # Uniquely sort the array
+  IFS=$'\n' local_files=($(sort -u <<<"${local_files[*]-}"))
+  unset IFS
+
+  for f in ${local_files[@]-}; do
+    # Find some copyright info in any file and break
+    egrep -wq "${ensure_pattern}" ${f} && \
+     FILE_CONTENT[${package}-${type}]="$(cat ${f})" && break
+  done
+  # When nothing is set at the package level, try package_root
+  : ${FILE_CONTENT[${package}-${type}]:="${FILE_CONTENT[${package_root}-${type}]-}"}
+
+  if [[ -z "${FILE_CONTENT[${package}-${type}]-}" ]]; then
+    for f in ${remote_files[@]}; do
+      file_state "${package_root_url}/master/${f}" && continue
+      if ! FILE_CONTENT[${package}-${type}]="$(\
+          curl --fail -s https://${package_root_url}/master/${f})" || \
+         ! $(echo "${FILE_CONTENT[${package}-${type}]-}" |\
+          egrep -qw "${ensure_pattern}") ||
+         [[ "${FILE_CONTENT[${package}-${type}]-}" =~ \<\ *html ]] ; then
+
+        ((CREATE_MISSING)) || file_state -a "${package_root_url}/master/${f}"
+        continue
+      fi
+
+      if [[ -n "${FILE_CONTENT[${package}-${type}]-}" ]]; then
+        if ((CREATE_MISSING)); then
+          mkdir -p ${GODEPS_SRC}/${package_root}
+          echo "${FILE_CONTENT[${package}-${type}]}" \
+           > ${GODEPS_SRC}/${package_root}/${f}
+        fi
+        break
+      fi
+    done
+  fi
+}
+
+
+#############################################################################
+# MAIN
+#############################################################################
+KUBE_ROOT="${KUBE_ROOT:-$(cd "$(dirname "${BASH_SOURCE}")/.." && pwd -P)}"
 cd "${KUBE_ROOT}"
 
-exec docker run --rm -i -v "${KUBE_ROOT}:/repo" mesosphere/godep-licenses:latest -p /repo \
-  -e github.com/abbot/go-http-auth:Apache-2 \
-  -e github.com/beorn7/perks/quantile:MIT? \
-  -e github.com/daviddengcn/go-colortext:BSD? \
-  -e github.com/docker/docker/pkg/symlink:spdxBSD3 \
-  -e github.com/shurcooL/sanitized_anchor_name:MIT? \
-  -e github.com/spf13/cobra:Apache-2 \
-  -e github.com/stretchr/objx:MIT? \
-  -e github.com/docker/spdystream:SeeFile \
-  -e gopkg.in/yaml.v2:LesserExceptionGPLVer3-TOOLONG \
-  -o md > Godeps/LICENSES.md
+# If CREATE_MISSING=1, the state file is ignored
+CREATE_MISSING=0
+if [[ ${1-} == "--create-missing" ]]; then
+  CREATE_MISSING=1
+  shift
+fi
+
+# Place to store the state of not-found files so we don't curl too much
+GODEPS_STATE="Godeps/.license_file_state"
+
+GODEPS_LICENSE_FILE=${1:-"Godeps/LICENSES"}
+GODEPS_SRC="Godeps/_workspace/src"
+declare -Ag FILE_CONTENT
+
+
+# Put the K8S LICENSE on top
+(
+echo "================================================================================"
+echo "= Kubernetes licensed under: ="
+echo
+cat ${KUBE_ROOT}/LICENSE
+) > ${GODEPS_LICENSE_FILE}
+
+# Loop through every package in Godeps.json
+for PACKAGE in $(cat Godeps/Godeps.json |\
+                 jq -r ".Deps[].ImportPath" |sort -f); do
+
+  process_content ${PACKAGE} LICENSE
+  process_content ${PACKAGE} COPYRIGHT
+
+  # display content
+  echo
+  echo "================================================================================"
+  echo "= ${GODEPS_SRC}/${PACKAGE} licensed under: ="
+  echo
+
+  if [[ -z "${FILE_CONTENT[${PACKAGE}-LICENSE]-}" &&
+        -z "${FILE_CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then
+    echo "UNKNOWN"
+  else
+    if [[ -n "${FILE_CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
+      echo "${FILE_CONTENT[${PACKAGE}-LICENSE]-}"
+      echo
+    fi
+    if [[ -n "${FILE_CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then
+      echo "${FILE_CONTENT[${PACKAGE}-COPYRIGHT]-}" | sed -n '/Copyright /,$p'
+    fi
+  fi
+done >> ${GODEPS_LICENSE_FILE}

hack/verify-godep-licenses.sh

@@ -36,13 +36,14 @@ function cleanup {
 }
 trap cleanup EXIT
 
+cp -r "${KUBE_ROOT}/LICENSE" "${_tmpdir}/"
 cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps"
 
 # Update Godep Licenses
 KUBE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-godep-licenses.sh"
 
 # Compare Godep Licenses
-if ! _out="$(diff -Naupr ${KUBE_ROOT}/Godeps/LICENSES.md ${_tmpdir}/Godeps/LICENSES.md)"; then
+if ! _out="$(diff -Naupr ${KUBE_ROOT}/Godeps/LICENSES ${_tmpdir}/Godeps/LICENSES)"; then
   echo "Your godep licenses file is out of date. Run hack/update-godep-licenses.sh and commit the results."
   echo "${_out}"
   exit 1