Implementing standalone etcd for openshift

examples/examples_test.go

@@ -283,8 +283,14 @@ func TestExampleObjectSchemas(t *testing.T) {
 			"pod": &api.Pod{},
 		},
 		"../examples/openshift-origin": {
-			"openshift-controller": &api.ReplicationController{},
-			"openshift-service":    &api.Service{},
+			"openshift-origin-namespace": &api.Namespace{},
+			"openshift-controller":       &api.ReplicationController{},
+			"openshift-service":          &api.Service{},
+			"etcd-controller":            &api.ReplicationController{},
+			"etcd-service":               &api.Service{},
+			"etcd-discovery-controller":  &api.ReplicationController{},
+			"etcd-discovery-service":     &api.Service{},
+			"secret":                     nil,
 		},
 		"../examples/phabricator": {
 			"authenticator-controller": &api.ReplicationController{},

examples/openshift-origin/.gitignore

 config/
-secret.json
-*.log

examples/openshift-origin/cleanup.sh

@@ -15,11 +15,29 @@
 # limitations under the License.
 
 # Cleans up resources from the example, assumed to be run from Kubernetes repo root
-
+echo
+echo
 export OPENSHIFT_EXAMPLE=$(pwd)/examples/openshift-origin
 export OPENSHIFT_CONFIG=${OPENSHIFT_EXAMPLE}/config
-rm -fr ${OPENSHIFT_CONFIG}
-cluster/kubectl.sh delete secrets openshift-config
-cluster/kubectl.sh stop rc openshift
-cluster/kubectl.sh delete rc openshift
-cluster/kubectl.sh delete services openshift
+
+echo "===> Removing the OpenShift namespace:"
+kubectl delete namespace openshift-origin
+echo
+
+echo "===> Removing local files:"
+rm -rf ${OPENSHIFT_CONFIG}
+rm ${OPENSHIFT_EXAMPLE}/openshift-startup.log
+rm ${OPENSHIFT_EXAMPLE}/secret.json
+touch ${OPENSHIFT_EXAMPLE}/secret.json
+echo
+
+echo "===> Restoring changed YAML specifcations:"
+if [ -f "${OPENSHIFT_EXAMPLE}/etcd-controller.yaml.bak" ]; then
+	rm ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml
+	mv -v ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml.bak ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml
+else
+	echo "No changed specifications found."
+fi
+echo
+
+echo Done.

examples/openshift-origin/create.sh

@@ -14,18 +14,108 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 # Creates resources from the example, assumed to be run from Kubernetes repo root
+echo
+echo "===> Initializing:"
+if [ ! $(which python) ]
+then
+	echo "Python is a prerequisite for running this script. Please install Python and try running again."
+	exit 1
+fi
+
+if [ ! $(which gcloud) ]
+then
+	echo "gcloud is a prerequisite for running this script. Please install gcloud and try running again."
+	exit 1
+fi
+
+gcloud_instances=$(gcloud compute instances list | grep "\-master")
+if [ -z "$gcloud_instances" ] || [ -z "${KUBE_GCE_INSTANCE_PREFIX}" ]
+then
+	echo "This script is only able to supply the necessary serviceaccount key if you are running on Google"
+	echo "Compute Engine using a cluster/kube-up.sh script with KUBE_GCE_INSTANCE_PREFIX set. If this is not"
+	echo "the case, be ready to supply a path to the serviceaccount public key."
+	if [ -z "${KUBE_GCE_INSTANCE_PREFIX}" ]
+	then
+		echo "Please provide your KUBE_GCE_INSTANCE_PREFIX now:"
+		read KUBE_GCE_INSTANCE_PREFIX
+	fi
+fi
+
 export OPENSHIFT_EXAMPLE=$(pwd)/examples/openshift-origin
+echo Set OPENSHIFT_EXAMPLE=${OPENSHIFT_EXAMPLE}
 export OPENSHIFT_CONFIG=${OPENSHIFT_EXAMPLE}/config
+echo Set OPENSHIFT_CONFIG=${OPENSHIFT_CONFIG}
 mkdir ${OPENSHIFT_CONFIG}
-cluster/kubectl.sh config view --output=yaml --flatten=true --minify=true > ${OPENSHIFT_CONFIG}/kubeconfig
-cluster/kubectl.sh create -f $OPENSHIFT_EXAMPLE/openshift-service.yaml
-sleep 60
-export PUBLIC_IP=$(cluster/kubectl.sh get services openshift --template="{{ index .status.loadBalancer.ingress 0 \"ip\" }}")
-echo "PUBLIC IP: ${PUBLIC_IP}"
-docker run --privileged -v ${OPENSHIFT_CONFIG}:/config openshift/origin start master --write-config=/config --kubeconfig=/config/kubeconfig --master=https://localhost:8443 --public-master=https://${PUBLIC_IP}:8443
-sudo -E chown ${USER} -R ${OPENSHIFT_CONFIG}
-docker run -i -t --privileged -e="OPENSHIFTCONFIG=/config/admin.kubeconfig" -v ${OPENSHIFT_CONFIG}:/config openshift/origin ex bundle-secret openshift-config -f /config &> ${OPENSHIFT_EXAMPLE}/secret.json
-cluster/kubectl.sh create -f ${OPENSHIFT_EXAMPLE}/secret.json
-cluster/kubectl.sh create -f ${OPENSHIFT_EXAMPLE}/openshift-controller.yaml
-cluster/kubectl.sh get pods | grep openshift
+echo Made dir ${OPENSHIFT_CONFIG}
+echo
+
+echo "===> Setting up OpenShift-Origin namespace:"
+kubectl create -f ${OPENSHIFT_EXAMPLE}/openshift-origin-namespace.yaml
+echo
+
+echo "===> Setting up etcd-discovery:"
+# A token etcd uses to generate unique cluster ID and member ID. Conforms to [a-z0-9]{40}
+export ETCD_INITIAL_CLUSTER_TOKEN=$(python -c "import string; import random; print(''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(40)))")
+
+# A unique token used by the discovery service. Conforms to etcd-cluster-[a-z0-9]{5}
+export ETCD_DISCOVERY_TOKEN=$(python -c "import string; import random; print(\"etcd-cluster-\" + ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(5)))")
+sed -i.bak -e "s/INSERT_ETCD_INITIAL_CLUSTER_TOKEN/\"${ETCD_INITIAL_CLUSTER_TOKEN}\"/g" -e "s/INSERT_ETCD_DISCOVERY_TOKEN/\"${ETCD_DISCOVERY_TOKEN}\"/g" ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml
+
+kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-discovery-controller.yaml --namespace='openshift-origin'
+kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-discovery-service.yaml --namespace='openshift-origin'
+echo
+
+echo "===> Setting up etcd:"
+kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml --namespace='openshift-origin'
+kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-service.yaml --namespace='openshift-origin'
+echo
+
+echo "===> Setting up openshift-origin:"
+kubectl config view --output=yaml --flatten=true --minify=true > ${OPENSHIFT_CONFIG}/kubeconfig
+kubectl create -f ${OPENSHIFT_EXAMPLE}/openshift-service.yaml --namespace='openshift-origin'
+echo
+
+export PUBLIC_OPENSHIFT_IP=""
+echo "===> Waiting for public IP to be set for the OpenShift Service."
+echo "Mistakes in service setup can cause this to loop infinitely if an"
+echo "external IP is never set. Ensure that the OpenShift service"
+echo "is set to use an external load balancer. This process may take" 
+echo "a few minutes. Errors can be found in the log file found at:"
+echo ${OPENSHIFT_EXAMPLE}/openshift-startup.log
+echo "" > ${OPENSHIFT_EXAMPLE}/openshift-startup.log
+while [ ${#PUBLIC_OPENSHIFT_IP} -lt 1 ]; do
+	echo -n .
+	sleep 1
+	{
+		export PUBLIC_OPENSHIFT_IP=$(kubectl get services openshift --namespace="openshift-origin" --template="{{ index .status.loadBalancer.ingress 0 \"ip\" }}")
+	} >> ${OPENSHIFT_EXAMPLE}/openshift-startup.log 2>&1
+	if [[ ! ${PUBLIC_OPENSHIFT_IP} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
+		export PUBLIC_OPENSHIFT_IP=""
+	fi
+done
+echo
+echo "Public OpenShift IP set to: ${PUBLIC_OPENSHIFT_IP}"
+echo
+
+echo "===> Configuring OpenShift:"
+docker run --privileged -v ${OPENSHIFT_CONFIG}:/config openshift/origin start master --write-config=/config --kubeconfig=/config/kubeconfig --master=https://localhost:8443 --public-master=https://${PUBLIC_OPENSHIFT_IP}:8443 --etcd=http://etcd:2379
+sudo -E chown -R ${USER} ${OPENSHIFT_CONFIG}
+
+# The following assumes GCE and that KUBE_GCE_INSTANCE_PREFIX is set
+export ZONE=$(gcloud compute instances list | grep "${KUBE_GCE_INSTANCE_PREFIX}\-master" | awk '{print $2}' | head -1)
+echo "sudo cat /srv/kubernetes/server.key; exit;" | gcloud compute ssh ${KUBE_GCE_INSTANCE_PREFIX}-master --zone ${ZONE} | grep -Ex "(^\-.*\-$|^\S+$)" > ${OPENSHIFT_CONFIG}/serviceaccounts.private.key
+# The following insertion will fail if indentation changes
+sed -i -e 's/publicKeyFiles:.*$/publicKeyFiles:/g' -e '/publicKeyFiles:/a \ \ - serviceaccounts.private.key' ${OPENSHIFT_CONFIG}/master-config.yaml
+
+docker run -it --privileged -e="KUBECONFIG=/config/admin.kubeconfig" -v ${OPENSHIFT_CONFIG}:/config openshift/origin cli secrets new openshift-config /config -o json &> ${OPENSHIFT_EXAMPLE}/secret.json
+kubectl create -f ${OPENSHIFT_EXAMPLE}/secret.json --namespace='openshift-origin'
+echo
+
+echo "===> Running OpenShift Master:"
+kubectl create -f ${OPENSHIFT_EXAMPLE}/openshift-controller.yaml --namespace='openshift-origin'
+echo
+
+echo Done.

examples/openshift-origin/etcd-controller.yaml

+kind: ReplicationController
+apiVersion: v1
+metadata:
+  name: etcd
+  creationTimestamp: 
+spec:
+  strategy:
+    type: Recreate
+    resources: {}
+  triggers:
+  - type: ConfigChange
+  replicas: 3
+  selector:
+    name: etcd
+  template:
+    metadata:
+      creationTimestamp: 
+      labels:
+        name: etcd
+    spec:
+      containers:
+      - name: member
+        image: openshift/etcd-20-centos7
+        ports:
+        - containerPort: 2379
+          protocol: TCP
+        - containerPort: 2380
+          protocol: TCP
+        env:
+          # ETCD_NUM_MEMBERS is the maximum number of members to launch (have to match with # of replicas)
+        - name: ETCD_NUM_MEMBERS
+          value: "3"
+        - name: ETCD_INITIAL_CLUSTER_STATE
+          value: "new"
+          # ETCD_INITIAL_CLUSTER_TOKEN is a token etcd uses to generate unique cluster ID and member ID. Conforms to [a-z0-9]{40}
+        - name: ETCD_INITIAL_CLUSTER_TOKEN
+          value: INSERT_ETCD_INITIAL_CLUSTER_TOKEN
+          # ETCD_DISCOVERY_TOKEN is a unique token used by the discovery service. Conforms to etcd-cluster-[a-z0-9]{5}
+        - name: ETCD_DISCOVERY_TOKEN
+          value: INSERT_ETCD_DISCOVERY_TOKEN
+          # ETCD_DISCOVERY_URL connects etcd instances together by storing a list of peer addresses, 
+          # metadata and the initial size of the cluster under a unique address
+        - name: ETCD_DISCOVERY_URL
+          value: "http://etcd-discovery:2379"
+        - name: ETCDCTL_PEERS
+          value: "http://etcd:2379"
+        resources: {}
+        terminationMessagePath: "/dev/termination-log"
+        imagePullPolicy: IfNotPresent
+        capabilities: {}
+        securityContext:
+          capabilities: {}
+          privileged: false
+      restartPolicy: Always
+      dnsPolicy: ClusterFirst
+      serviceAccount: ''
+status: {}

examples/openshift-origin/etcd-discovery-controller.yaml

+kind: ReplicationController
+apiVersion: v1
+metadata:
+  name: etcd-discovery
+  creationTimestamp: 
+spec:
+  strategy:
+    type: Recreate
+    resources: {}
+  triggers:
+  - type: ConfigChange
+  replicas: 1
+  selector:
+    name: etcd-discovery
+  template:
+    metadata:
+      creationTimestamp: 
+      labels:
+        name: etcd-discovery
+    spec:
+      containers:
+      - name: discovery
+        image: openshift/etcd-20-centos7
+        args:
+        - etcd-discovery.sh
+        ports:
+        - containerPort: 2379
+          protocol: TCP
+        resources: {}
+        terminationMessagePath: "/dev/termination-log"
+        imagePullPolicy: IfNotPresent
+        capabilities: {}
+        securityContext:
+          capabilities: {}
+          privileged: false
+      restartPolicy: Always
+      dnsPolicy: ClusterFirst
+      serviceAccount: ''
+status: {}

examples/openshift-origin/etcd-discovery-service.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: etcd-discovery
+  creationTimestamp: 
+  labels:
+    name: etcd-discovery
+spec:
+  ports:
+  - protocol: TCP
+    port: 2379
+    targetPort: 2379
+    nodePort: 0
+  selector:
+    name: etcd-discovery
+  sessionAffinity: None
+  type: ClusterIP
+status:
+  loadBalancer: {}

examples/openshift-origin/etcd-service.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: etcd
+  creationTimestamp: 
+  labels:
+    name: etcd
+spec:
+  ports:
+  - name: client
+    protocol: TCP
+    port: 2379
+    targetPort: 2379
+    nodePort: 0
+  - name: server
+    protocol: TCP
+    port: 2380
+    targetPort: 2380
+    nodePort: 0
+  selector:
+    name: etcd
+  sessionAffinity: None
+  type: ClusterIP
+status:
+  loadBalancer: {}

examples/openshift-origin/openshift-origin-namespace.yaml

+kind: Namespace
+apiVersion: v1
+metadata:
+  name: "openshift-origin"
+  labels:
+    name: "openshift-origin"
\ No newline at end of file

examples/openshift-origin/openshift-service.yaml

@@ -4,8 +4,8 @@ metadata:
   name: openshift
 spec:
   ports:
-    - port: 8443
-      name: openshift
+    - name: openshift
+      port: 8443
       targetPort: 8443
   selector:
     name: openshift

tmp-valid-pod.json

+{
+    "kind": "Pod",
+    "apiVersion": "v1",
+    "metadata": {
+        "name": "valid-pod",
+        "namespace": "default",
+        "selfLink": "/api/v1/namespaces/default/pods/valid-pod",
+        "uid": "20f4f1f5-1e67-11e5-b84d-54ee753e2644",
+        "resourceVersion": "474",
+        "creationTimestamp": "2015-06-29T13:59:58Z",
+        "labels": {
+            "name": "valid-pod"
+        }
+    },
+    "spec": {
+        "containers": [
+            {
+                "name": "update-k8s-serve-hostname",
+                "image": "nginx",
+                "resources": {
+                    "limits": {
+                        "cpu": "1",
+                        "memory": "6Mi"
+                    }
+                },
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent"
+            }
+        ],
+        "restartPolicy": "Always",
+        "dnsPolicy": "ClusterFirst"
+    },
+    "status": {
+        "phase": "Pending"
+    }
+}