Merge pull request #58828 from rajansandeep/configtranslate

Godeps/Godeps.json

@@ -2093,6 +2093,11 @@
 			"Rev": "fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a"
 		},
 		{
+			"ImportPath": "github.com/mholt/caddy/caddyfile",
+			"Comment": "v0.10.10-57-g2de4950",
+			"Rev": "2de495001514ed50eac2e09f474c32d417100c5c"
+		},
+		{
 			"ImportPath": "github.com/miekg/dns",
 			"Rev": "5d001d020961ae1c184f9f8152fdc73810481677"
 		},

cmd/kubeadm/app/features/features.go

@@ -54,7 +54,7 @@ var InitFeatureGates = FeatureList{
 	StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}},
 	// We don't want to advertise this feature gate exists in v1.9 to avoid confusion as it is not yet working
 	HighAvailability:     {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190, HiddenInHelpText: true},
-	CoreDNS:              {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
+	CoreDNS:              {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}, MinimumVersion: v190},
 	DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
 	Auditing:             {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}},
 }

cmd/kubeadm/app/phases/addons/dns/BUILD

@@ -18,7 +18,9 @@ go_test(
         "//cmd/kubeadm/app/util:go_default_library",
         "//pkg/apis/core:go_default_library",
         "//pkg/util/version:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//vendor/k8s.io/client-go/testing:go_default_library",
@@ -41,6 +43,7 @@ go_library(
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/util/version:go_default_library",
+        "//vendor/github.com/mholt/caddy/caddyfile:go_default_library",
         "//vendor/k8s.io/api/apps/v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/rbac/v1:go_default_library",

cmd/kubeadm/app/phases/addons/dns/dns.go

@@ -17,8 +17,12 @@ limitations under the License.
 package dns
 
 import (
+	"encoding/json"
 	"fmt"
 	"runtime"
+	"strings"
+
+	"github.com/mholt/caddy/caddyfile"
 
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
@@ -38,7 +42,10 @@ import (
 
 const (
 	// KubeDNSServiceAccountName describes the name of the ServiceAccount for the kube-dns addon
-	KubeDNSServiceAccountName = "kube-dns"
+	KubeDNSServiceAccountName  = "kube-dns"
+	kubeDNSStubDomain          = "stubDomains"
+	kubeDNSUpstreamNameservers = "upstreamNameservers"
+	kubeDNSFederation          = "federations"
 )
 
 // EnsureDNSAddon creates the kube-dns or CoreDNS addon
@@ -140,10 +147,33 @@ func coreDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interfac
 		return fmt.Errorf("error when parsing CoreDNS deployment template: %v", err)
 	}
 
+	// Get the kube-dns ConfigMap for translation to equivalent CoreDNS Config.
+	kubeDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeDNS, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	stubDomain, err := translateStubDomainOfKubeDNSToProxyCoreDNS(kubeDNSStubDomain, kubeDNSConfigMap)
+	if err != nil {
+		return err
+	}
+
+	upstreamNameserver, err := translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS(kubeDNSUpstreamNameservers, kubeDNSConfigMap)
+	if err != nil {
+		return err
+	}
+	coreDNSDomain := cfg.Networking.DNSDomain
+	federations, err := translateFederationsofKubeDNSToCoreDNS(kubeDNSFederation, coreDNSDomain, kubeDNSConfigMap)
+	if err != nil {
+		return err
+	}
+
 	// Get the config file for CoreDNS
-	coreDNSConfigMapBytes, err := kubeadmutil.ParseTemplate(CoreDNSConfigMap, struct{ DNSDomain, ServiceCIDR string }{
-		ServiceCIDR: cfg.Networking.ServiceSubnet,
-		DNSDomain:   cfg.Networking.DNSDomain,
+	coreDNSConfigMapBytes, err := kubeadmutil.ParseTemplate(CoreDNSConfigMap, struct{ DNSDomain, UpstreamNameserver, Federation, StubDomain string }{
+		DNSDomain:          coreDNSDomain,
+		UpstreamNameserver: upstreamNameserver,
+		Federation:         federations,
+		StubDomain:         stubDomain,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing CoreDNS configMap template: %v", err)
@@ -244,3 +274,105 @@ func createDNSService(dnsService *v1.Service, serviceBytes []byte, client client
 	}
 	return nil
 }
+
+// translateStubDomainOfKubeDNSToProxyCoreDNS translates StubDomain Data in kube-dns ConfigMap
+// in the form of Proxy for the CoreDNS Corefile.
+func translateStubDomainOfKubeDNSToProxyCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
+	if proxy, ok := kubeDNSConfigMap.Data[dataField]; ok {
+		stubDomainData := make(map[string][]string)
+		err := json.Unmarshal([]byte(proxy), &stubDomainData)
+		if err != nil {
+			return "", fmt.Errorf("failed to parse JSON from 'kube-dns ConfigMap: %v", err)
+		}
+
+		var proxyStanza []interface{}
+		for domain, proxyIP := range stubDomainData {
+			pStanza := map[string]interface{}{}
+			pStanza["keys"] = []string{domain + ":53"}
+			pStanza["body"] = [][]string{
+				{"errors"},
+				{"cache", "30"},
+				append([]string{"proxy", "."}, proxyIP...),
+			}
+			proxyStanza = append(proxyStanza, pStanza)
+		}
+		stanzasBytes, err := json.Marshal(proxyStanza)
+		if err != nil {
+			return "", err
+		}
+
+		corefileStanza, err := caddyfile.FromJSON(stanzasBytes)
+		if err != nil {
+			return "", err
+		}
+
+		return prepCorefileFormat(string(corefileStanza), 4), nil
+	}
+	return "", nil
+}
+
+// translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS translates UpstreamNameServer Data in kube-dns ConfigMap
+// in the form of Proxy for the CoreDNS Corefile.
+func translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
+	if upstreamValues, ok := kubeDNSConfigMap.Data[dataField]; ok {
+		var upstreamProxyIP []string
+
+		err := json.Unmarshal([]byte(upstreamValues), &upstreamProxyIP)
+		if err != nil {
+			return "", fmt.Errorf("failed to parse JSON from 'kube-dns ConfigMap: %v", err)
+		}
+
+		coreDNSProxyStanzaList := strings.Join(upstreamProxyIP, " ")
+		return coreDNSProxyStanzaList, nil
+	}
+	return "/etc/resolv.conf", nil
+}
+
+// translateFederationsofKubeDNSToCoreDNS translates Federations Data in kube-dns ConfigMap
+// to Federation for CoreDNS Corefile.
+func translateFederationsofKubeDNSToCoreDNS(dataField, coreDNSDomain string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
+	if federation, ok := kubeDNSConfigMap.Data[dataField]; ok {
+		var (
+			federationStanza []interface{}
+			body             [][]string
+		)
+		federationData := make(map[string]string)
+
+		err := json.Unmarshal([]byte(federation), &federationData)
+		if err != nil {
+			return "", fmt.Errorf("failed to parse JSON from kube-dns ConfigMap: %v", err)
+		}
+		fStanza := map[string]interface{}{}
+
+		for name, domain := range federationData {
+			body = append(body, []string{name, domain})
+		}
+		federationStanza = append(federationStanza, fStanza)
+		fStanza["keys"] = []string{"federation " + coreDNSDomain}
+		fStanza["body"] = body
+		stanzasBytes, err := json.Marshal(federationStanza)
+		if err != nil {
+			return "", err
+		}
+
+		corefileStanza, err := caddyfile.FromJSON(stanzasBytes)
+		if err != nil {
+			return "", err
+		}
+
+		return prepCorefileFormat(string(corefileStanza), 8), nil
+	}
+	return "", nil
+}
+
+// prepCorefileFormat indents the output of the Corefile caddytext and replaces tabs with spaces
+// to neatly format the configmap, making it readable.
+func prepCorefileFormat(s string, indentation int) string {
+	r := []string{}
+	for _, line := range strings.Split(s, "\n") {
+		indented := strings.Repeat(" ", indentation) + line
+		r = append(r, indented)
+	}
+	corefile := strings.Join(r, "\n")
+	return "\n" + strings.Replace(corefile, "\t", "   ", -1)
+}

cmd/kubeadm/app/phases/addons/dns/dns_test.go

@@ -17,9 +17,12 @@ limitations under the License.
 package dns
 
 import (
+	"strings"
 	"testing"
 
+	"k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientsetfake "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
@@ -127,9 +130,11 @@ func TestCompileManifests(t *testing.T) {
 		},
 		{
 			manifest: CoreDNSConfigMap,
-			data: struct{ DNSDomain, ServiceCIDR string }{
-				DNSDomain:   "foo",
-				ServiceCIDR: "foo",
+			data: struct{ DNSDomain, Federation, UpstreamNameserver, StubDomain string }{
+				DNSDomain:          "foo",
+				Federation:         "foo",
+				UpstreamNameserver: "foo",
+				StubDomain:         "foo",
 			},
 			expected: true,
 		},
@@ -175,3 +180,238 @@ func TestGetDNSIP(t *testing.T) {
 		}
 	}
 }
+
+func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
+	testCases := []struct {
+		configMap *v1.ConfigMap
+		expectOne string
+		expectTwo string
+	}{
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kube-dns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"stubDomains":         `{"foo.com" : ["1.2.3.4:5300","3.3.3.3"], "my.cluster.local" : ["2.3.4.5"]}`,
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
+				},
+			},
+
+			expectOne: `
+    foo.com:53 {
+       errors
+       cache 30
+       proxy . 1.2.3.4:5300 3.3.3.3
+    }
+    
+    my.cluster.local:53 {
+       errors
+       cache 30
+       proxy . 2.3.4.5
+    }`,
+			expectTwo: `
+    my.cluster.local:53 {
+       errors
+       cache 30
+       proxy . 2.3.4.5
+    }
+    
+    foo.com:53 {
+       errors
+       cache 30
+       proxy . 1.2.3.4:5300 3.3.3.3
+    }`,
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kubedns",
+					Namespace: "kube-system",
+				},
+			},
+
+			expectOne: "",
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kube-dns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"stubDomains":         `{"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`,
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
+				},
+			},
+
+			expectOne: `
+    foo.com:53 {
+       errors
+       cache 30
+       proxy . 1.2.3.4:5300
+    }
+    
+    my.cluster.local:53 {
+       errors
+       cache 30
+       proxy . 2.3.4.5
+    }`,
+			expectTwo: `
+    my.cluster.local:53 {
+       errors
+       cache 30
+       proxy . 2.3.4.5
+    }
+    
+    foo.com:53 {
+       errors
+       cache 30
+       proxy . 1.2.3.4:5300
+    }`,
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kube-dns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
+				},
+			},
+
+			expectOne: "",
+		},
+	}
+	for _, testCase := range testCases {
+		out, err := translateStubDomainOfKubeDNSToProxyCoreDNS(kubeDNSStubDomain, testCase.configMap)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if !strings.Contains(out, testCase.expectOne) && !strings.Contains(out, testCase.expectTwo) {
+			t.Errorf("expected to find %q or %q in output: %q", testCase.expectOne, testCase.expectTwo, out)
+		}
+	}
+}
+
+func TestTranslateUpstreamKubeDNSToCoreDNS(t *testing.T) {
+	testCases := []struct {
+		configMap *v1.ConfigMap
+		expect    string
+	}{
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kube-dns",
+					Namespace: "kube-system",
+				},
+			},
+
+			expect: "/etc/resolv.conf",
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kubedns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"stubDomains":         ` {"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`,
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4", "4.4.4.4"]`,
+				},
+			},
+
+			expect: "8.8.8.8 8.8.4.4 4.4.4.4",
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kubedns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
+				},
+			},
+
+			expect: "8.8.8.8 8.8.4.4",
+		},
+	}
+	for _, testCase := range testCases {
+		out, err := translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS(kubeDNSUpstreamNameservers, testCase.configMap)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if !strings.Contains(out, testCase.expect) {
+			t.Errorf("expected to find %q in output: %q", testCase.expect, out)
+		}
+	}
+}
+
+func TestTranslateFederationKubeDNSToCoreDNS(t *testing.T) {
+	testCases := []struct {
+		configMap *v1.ConfigMap
+		expectOne string
+		expectTwo string
+	}{
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kube-dns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"federations":         `{"foo" : "foo.feddomain.com", "bar" : "bar.feddomain.com"}`,
+					"stubDomains":         `{"foo.com" : ["1.2.3.4:5300","3.3.3.3"], "my.cluster.local" : ["2.3.4.5"]}`,
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
+				},
+			},
+
+			expectOne: `
+        federation cluster.local {
+           foo foo.feddomain.com
+           bar bar.feddomain.com
+        }`,
+			expectTwo: `
+        federation cluster.local {
+           bar bar.feddomain.com
+           foo foo.feddomain.com
+        }`,
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kubedns",
+					Namespace: "kube-system",
+				},
+			},
+
+			expectOne: "",
+		},
+		{
+			configMap: &v1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "kube-dns",
+					Namespace: "kube-system",
+				},
+				Data: map[string]string{
+					"stubDomains":         `{"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`,
+					"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
+				},
+			},
+
+			expectOne: "",
+		},
+	}
+	for _, testCase := range testCases {
+		out, err := translateFederationsofKubeDNSToCoreDNS(kubeDNSFederation, "cluster.local", testCase.configMap)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if !strings.Contains(out, testCase.expectOne) && !strings.Contains(out, testCase.expectTwo) {
+			t.Errorf("expected to find %q or %q in output: %q", testCase.expectOne, testCase.expectTwo, out)
+		}
+	}
+}

cmd/kubeadm/app/phases/addons/dns/manifests.go

@@ -307,15 +307,15 @@ data:
     .:53 {
         errors
         health
-        kubernetes {{ .DNSDomain }} {{ .ServiceCIDR }} {
+        kubernetes {{ .DNSDomain }} in-addr.arpa ip6.arpa {
            pods insecure
-           upstream /etc/resolv.conf
+           upstream
            fallthrough in-addr.arpa ip6.arpa
-        }
+        }{{ .Federation }}
         prometheus :9153
-        proxy . /etc/resolv.conf
+        proxy . {{ .UpstreamNameserver }}
         cache 30
-    }
+    }{{ .StubDomain }}
 `
 	// CoreDNSClusterRole is the CoreDNS ClusterRole manifest
 	CoreDNSClusterRole = `

cmd/kubeadm/app/phases/addons/dns/versions.go

@@ -23,13 +23,13 @@ import (
 
 const (
 	kubeDNSv190AndAboveVersion = "1.14.8"
-	coreDNSVersion             = "1.0.4"
+	coreDNSVersion             = "1.0.6"
 )
 
 // GetDNSVersion returns the right kube-dns version for a specific k8s version
 func GetDNSVersion(kubeVersion *version.Version, dns string) string {
 	// v1.9.0+ uses kube-dns 1.14.8
-	// v1.9.0+ uses CoreDNS  1.0.4 if feature gate "CoreDNS" is enabled.
+	// v1.9.0+ uses CoreDNS  1.0.6 if feature gate "CoreDNS" is enabled.
 
 	// In the future when the version is bumped at HEAD; add conditional logic to return the right versions
 	// Also, the version might be bumped for different k8s releases on the same branch

vendor/BUILD

@@ -286,6 +286,7 @@ filegroup(
         "//vendor/github.com/mailru/easyjson/jwriter:all-srcs",
         "//vendor/github.com/marstr/guid:all-srcs",
         "//vendor/github.com/matttproud/golang_protobuf_extensions/pbutil:all-srcs",
+        "//vendor/github.com/mholt/caddy/caddyfile:all-srcs",
         "//vendor/github.com/miekg/dns:all-srcs",
         "//vendor/github.com/mindprince/gonvml:all-srcs",
         "//vendor/github.com/mistifyio/go-zfs:all-srcs",

vendor/github.com/mholt/caddy/caddyfile/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dispenser.go",
+        "json.go",
+        "lexer.go",
+        "parse.go",
+    ],
+    importpath = "github.com/mholt/caddy/caddyfile",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/github.com/mholt/caddy/caddyfile/dispenser.go

+// Copyright 2015 Light Code Labs, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyfile
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Dispenser is a type that dispenses tokens, similarly to a lexer,
+// except that it can do so with some notion of structure and has
+// some really convenient methods.
+type Dispenser struct {
+	filename string
+	tokens   []Token
+	cursor   int
+	nesting  int
+}
+
+// NewDispenser returns a Dispenser, ready to use for parsing the given input.
+func NewDispenser(filename string, input io.Reader) Dispenser {
+	tokens, _ := allTokens(input) // ignoring error because nothing to do with it
+	return Dispenser{
+		filename: filename,
+		tokens:   tokens,
+		cursor:   -1,
+	}
+}
+
+// NewDispenserTokens returns a Dispenser filled with the given tokens.
+func NewDispenserTokens(filename string, tokens []Token) Dispenser {
+	return Dispenser{
+		filename: filename,
+		tokens:   tokens,
+		cursor:   -1,
+	}
+}
+
+// Next loads the next token. Returns true if a token
+// was loaded; false otherwise. If false, all tokens
+// have been consumed.
+func (d *Dispenser) Next() bool {
+	if d.cursor < len(d.tokens)-1 {
+		d.cursor++
+		return true
+	}
+	return false
+}
+
+// NextArg loads the next token if it is on the same
+// line. Returns true if a token was loaded; false
+// otherwise. If false, all tokens on the line have
+// been consumed. It handles imported tokens correctly.
+func (d *Dispenser) NextArg() bool {
+	if d.cursor < 0 {
+		d.cursor++
+		return true
+	}
+	if d.cursor >= len(d.tokens) {
+		return false
+	}
+	if d.cursor < len(d.tokens)-1 &&
+		d.tokens[d.cursor].File == d.tokens[d.cursor+1].File &&
+		d.tokens[d.cursor].Line+d.numLineBreaks(d.cursor) == d.tokens[d.cursor+1].Line {
+		d.cursor++
+		return true
+	}
+	return false
+}
+
+// NextLine loads the next token only if it is not on the same
+// line as the current token, and returns true if a token was
+// loaded; false otherwise. If false, there is not another token
+// or it is on the same line. It handles imported tokens correctly.
+func (d *Dispenser) NextLine() bool {
+	if d.cursor < 0 {
+		d.cursor++
+		return true
+	}
+	if d.cursor >= len(d.tokens) {
+		return false
+	}
+	if d.cursor < len(d.tokens)-1 &&
+		(d.tokens[d.cursor].File != d.tokens[d.cursor+1].File ||
+			d.tokens[d.cursor].Line+d.numLineBreaks(d.cursor) < d.tokens[d.cursor+1].Line) {
+		d.cursor++
+		return true
+	}
+	return false
+}
+
+// NextBlock can be used as the condition of a for loop
+// to load the next token as long as it opens a block or
+// is already in a block. It returns true if a token was
+// loaded, or false when the block's closing curly brace
+// was loaded and thus the block ended. Nested blocks are
+// not supported.
+func (d *Dispenser) NextBlock() bool {
+	if d.nesting > 0 {
+		d.Next()
+		if d.Val() == "}" {
+			d.nesting--
+			return false
+		}
+		return true
+	}
+	if !d.NextArg() { // block must open on same line
+		return false
+	}
+	if d.Val() != "{" {
+		d.cursor-- // roll back if not opening brace
+		return false
+	}
+	d.Next()
+	if d.Val() == "}" {
+		// Open and then closed right away
+		return false
+	}
+	d.nesting++
+	return true
+}
+
+// Val gets the text of the current token. If there is no token
+// loaded, it returns empty string.
+func (d *Dispenser) Val() string {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return ""
+	}
+	return d.tokens[d.cursor].Text
+}
+
+// Line gets the line number of the current token. If there is no token
+// loaded, it returns 0.
+func (d *Dispenser) Line() int {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return 0
+	}
+	return d.tokens[d.cursor].Line
+}
+
+// File gets the filename of the current token. If there is no token loaded,
+// it returns the filename originally given when parsing started.
+func (d *Dispenser) File() string {
+	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+		return d.filename
+	}
+	if tokenFilename := d.tokens[d.cursor].File; tokenFilename != "" {
+		return tokenFilename
+	}
+	return d.filename
+}
+
+// Args is a convenience function that loads the next arguments
+// (tokens on the same line) into an arbitrary number of strings
+// pointed to in targets. If there are fewer tokens available
+// than string pointers, the remaining strings will not be changed
+// and false will be returned. If there were enough tokens available
+// to fill the arguments, then true will be returned.
+func (d *Dispenser) Args(targets ...*string) bool {
+	enough := true
+	for i := 0; i < len(targets); i++ {
+		if !d.NextArg() {
+			enough = false
+			break
+		}
+		*targets[i] = d.Val()
+	}
+	return enough
+}
+
+// RemainingArgs loads any more arguments (tokens on the same line)
+// into a slice and returns them. Open curly brace tokens also indicate
+// the end of arguments, and the curly brace is not included in
+// the return value nor is it loaded.
+func (d *Dispenser) RemainingArgs() []string {
+	var args []string
+
+	for d.NextArg() {
+		if d.Val() == "{" {
+			d.cursor--
+			break
+		}
+		args = append(args, d.Val())
+	}
+
+	return args
+}
+
+// ArgErr returns an argument error, meaning that another
+// argument was expected but not found. In other words,
+// a line break or open curly brace was encountered instead of
+// an argument.
+func (d *Dispenser) ArgErr() error {
+	if d.Val() == "{" {
+		return d.Err("Unexpected token '{', expecting argument")
+	}
+	return d.Errf("Wrong argument count or unexpected line ending after '%s'", d.Val())
+}
+
+// SyntaxErr creates a generic syntax error which explains what was
+// found and what was expected.
+func (d *Dispenser) SyntaxErr(expected string) error {
+	msg := fmt.Sprintf("%s:%d - Syntax error: Unexpected token '%s', expecting '%s'", d.File(), d.Line(), d.Val(), expected)
+	return errors.New(msg)
+}
+
+// EOFErr returns an error indicating that the dispenser reached
+// the end of the input when searching for the next token.
+func (d *Dispenser) EOFErr() error {
+	return d.Errf("Unexpected EOF")
+}
+
+// Err generates a custom parse-time error with a message of msg.
+func (d *Dispenser) Err(msg string) error {
+	msg = fmt.Sprintf("%s:%d - Error during parsing: %s", d.File(), d.Line(), msg)
+	return errors.New(msg)
+}
+
+// Errf is like Err, but for formatted error messages
+func (d *Dispenser) Errf(format string, args ...interface{}) error {
+	return d.Err(fmt.Sprintf(format, args...))
+}
+
+// numLineBreaks counts how many line breaks are in the token
+// value given by the token index tknIdx. It returns 0 if the
+// token does not exist or there are no line breaks.
+func (d *Dispenser) numLineBreaks(tknIdx int) int {
+	if tknIdx < 0 || tknIdx >= len(d.tokens) {
+		return 0
+	}
+	return strings.Count(d.tokens[tknIdx].Text, "\n")
+}
+
+// isNewLine determines whether the current token is on a different
+// line (higher line number) than the previous token. It handles imported
+// tokens correctly. If there isn't a previous token, it returns true.
+func (d *Dispenser) isNewLine() bool {
+	if d.cursor < 1 {
+		return true
+	}
+	if d.cursor > len(d.tokens)-1 {
+		return false
+	}
+	return d.tokens[d.cursor-1].File != d.tokens[d.cursor].File ||
+		d.tokens[d.cursor-1].Line+d.numLineBreaks(d.cursor-1) < d.tokens[d.cursor].Line
+}

vendor/github.com/mholt/caddy/caddyfile/json.go

+// Copyright 2015 Light Code Labs, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyfile
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+const filename = "Caddyfile"
+
+// ToJSON converts caddyfile to its JSON representation.
+func ToJSON(caddyfile []byte) ([]byte, error) {
+	var j EncodedCaddyfile
+
+	serverBlocks, err := Parse(filename, bytes.NewReader(caddyfile), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, sb := range serverBlocks {
+		block := EncodedServerBlock{
+			Keys: sb.Keys,
+			Body: [][]interface{}{},
+		}
+
+		// Extract directives deterministically by sorting them
+		var directives = make([]string, len(sb.Tokens))
+		for dir := range sb.Tokens {
+			directives = append(directives, dir)
+		}
+		sort.Strings(directives)
+
+		// Convert each directive's tokens into our JSON structure
+		for _, dir := range directives {
+			disp := NewDispenserTokens(filename, sb.Tokens[dir])
+			for disp.Next() {
+				block.Body = append(block.Body, constructLine(&disp))
+			}
+		}
+
+		// tack this block onto the end of the list
+		j = append(j, block)
+	}
+
+	result, err := json.Marshal(j)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+// constructLine transforms tokens into a JSON-encodable structure;
+// but only one line at a time, to be used at the top-level of
+// a server block only (where the first token on each line is a
+// directive) - not to be used at any other nesting level.
+func constructLine(d *Dispenser) []interface{} {
+	var args []interface{}
+
+	args = append(args, d.Val())
+
+	for d.NextArg() {
+		if d.Val() == "{" {
+			args = append(args, constructBlock(d))
+			continue
+		}
+		args = append(args, d.Val())
+	}
+
+	return args
+}
+
+// constructBlock recursively processes tokens into a
+// JSON-encodable structure. To be used in a directive's
+// block. Goes to end of block.
+func constructBlock(d *Dispenser) [][]interface{} {
+	block := [][]interface{}{}
+
+	for d.Next() {
+		if d.Val() == "}" {
+			break
+		}
+		block = append(block, constructLine(d))
+	}
+
+	return block
+}
+
+// FromJSON converts JSON-encoded jsonBytes to Caddyfile text
+func FromJSON(jsonBytes []byte) ([]byte, error) {
+	var j EncodedCaddyfile
+	var result string
+
+	err := json.Unmarshal(jsonBytes, &j)
+	if err != nil {
+		return nil, err
+	}
+
+	for sbPos, sb := range j {
+		if sbPos > 0 {
+			result += "\n\n"
+		}
+		for i, key := range sb.Keys {
+			if i > 0 {
+				result += ", "
+			}
+			//result += standardizeScheme(key)
+			result += key
+		}
+		result += jsonToText(sb.Body, 1)
+	}
+
+	return []byte(result), nil
+}
+
+// jsonToText recursively transforms a scope of JSON into plain
+// Caddyfile text.
+func jsonToText(scope interface{}, depth int) string {
+	var result string
+
+	switch val := scope.(type) {
+	case string:
+		if strings.ContainsAny(val, "\" \n\t\r") {
+			result += `"` + strings.Replace(val, "\"", "\\\"", -1) + `"`
+		} else {
+			result += val
+		}
+	case int:
+		result += strconv.Itoa(val)
+	case float64:
+		result += fmt.Sprintf("%v", val)
+	case bool:
+		result += fmt.Sprintf("%t", val)
+	case [][]interface{}:
+		result += " {\n"
+		for _, arg := range val {
+			result += strings.Repeat("\t", depth) + jsonToText(arg, depth+1) + "\n"
+		}
+		result += strings.Repeat("\t", depth-1) + "}"
+	case []interface{}:
+		for i, v := range val {
+			if block, ok := v.([]interface{}); ok {
+				result += "{\n"
+				for _, arg := range block {
+					result += strings.Repeat("\t", depth) + jsonToText(arg, depth+1) + "\n"
+				}
+				result += strings.Repeat("\t", depth-1) + "}"
+				continue
+			}
+			result += jsonToText(v, depth)
+			if i < len(val)-1 {
+				result += " "
+			}
+		}
+	}
+
+	return result
+}
+
+// TODO: Will this function come in handy somewhere else?
+/*
+// standardizeScheme turns an address like host:https into https://host,
+// or "host:" into "host".
+func standardizeScheme(addr string) string {
+	if hostname, port, err := net.SplitHostPort(addr); err == nil {
+		if port == "http" || port == "https" {
+			addr = port + "://" + hostname
+		}
+	}
+	return strings.TrimSuffix(addr, ":")
+}
+*/
+
+// EncodedCaddyfile encapsulates a slice of EncodedServerBlocks.
+type EncodedCaddyfile []EncodedServerBlock
+
+// EncodedServerBlock represents a server block ripe for encoding.
+type EncodedServerBlock struct {
+	Keys []string        `json:"keys"`
+	Body [][]interface{} `json:"body"`
+}

vendor/github.com/mholt/caddy/caddyfile/lexer.go

+// Copyright 2015 Light Code Labs, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddyfile
+
+import (
+	"bufio"
+	"io"
+	"unicode"
+)
+
+type (
+	// lexer is a utility which can get values, token by
+	// token, from a Reader. A token is a word, and tokens
+	// are separated by whitespace. A word can be enclosed
+	// in quotes if it contains whitespace.
+	lexer struct {
+		reader *bufio.Reader
+		token  Token
+		line   int
+	}
+
+	// Token represents a single parsable unit.
+	Token struct {
+		File string
+		Line int
+		Text string
+	}
+)
+
+// load prepares the lexer to scan an input for tokens.
+// It discards any leading byte order mark.
+func (l *lexer) load(input io.Reader) error {
+	l.reader = bufio.NewReader(input)
+	l.line = 1
+
+	// discard byte order mark, if present
+	firstCh, _, err := l.reader.ReadRune()
+	if err != nil {
+		return err
+	}
+	if firstCh != 0xFEFF {
+		err := l.reader.UnreadRune()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// next loads the next token into the lexer.
+// A token is delimited by whitespace, unless
+// the token starts with a quotes character (")
+// in which case the token goes until the closing
+// quotes (the enclosing quotes are not included).
+// Inside quoted strings, quotes may be escaped
+// with a preceding \ character. No other chars
+// may be escaped. The rest of the line is skipped
+// if a "#" character is read in. Returns true if
+// a token was loaded; false otherwise.
+func (l *lexer) next() bool {
+	var val []rune
+	var comment, quoted, escaped bool
+
+	makeToken := func() bool {
+		l.token.Text = string(val)
+		return true
+	}
+
+	for {
+		ch, _, err := l.reader.ReadRune()
+		if err != nil {
+			if len(val) > 0 {
+				return makeToken()
+			}
+			if err == io.EOF {
+				return false
+			}
+			panic(err)
+		}
+
+		if quoted {
+			if !escaped {
+				if ch == '\\' {
+					escaped = true
+					continue
+				} else if ch == '"' {
+					quoted = false
+					return makeToken()
+				}
+			}
+			if ch == '\n' {
+				l.line++
+			}
+			if escaped {
+				// only escape quotes
+				if ch != '"' {
+					val = append(val, '\\')
+				}
+			}
+			val = append(val, ch)
+			escaped = false
+			continue
+		}
+
+		if unicode.IsSpace(ch) {
+			if ch == '\r' {
+				continue
+			}
+			if ch == '\n' {
+				l.line++
+				comment = false
+			}
+			if len(val) > 0 {
+				return makeToken()
+			}
+			continue
+		}
+
+		if ch == '#' {
+			comment = true
+		}
+
+		if comment {
+			continue
+		}
+
+		if len(val) == 0 {
+			l.token = Token{Line: l.line}
+			if ch == '"' {
+				quoted = true
+				continue
+			}
+		}
+
+		val = append(val, ch)
+	}
+}