Retry methods and set states so we immediately report config changes.

cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py

@@ -17,6 +17,8 @@
 import re
 import subprocess
 
+from time import sleep
+
 
 def get_version(bin_name):
     """Get the version of an installed Kubernetes binary.
@@ -33,3 +35,37 @@ def get_version(bin_name):
     cmd = '{} --version'.format(bin_name).split()
     version_string = subprocess.check_output(cmd).decode('utf-8')
     return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3])
+
+
+def retry(times, delay_secs):
+    """ Decorator for retrying a method call.
+
+    Args:
+        times: How many times should we retry before giving up
+        delay_secs: Delay in secs
+
+    Returns: A callable that would return the last call outcome
+    """
+
+    def retry_decorator(func):
+        """ Decorator to wrap the function provided.
+
+        Args:
+            func: Provided function should return either True od False
+
+        Returns: A callable that would return the last call outcome
+
+        """
+        def _wrapped(*args, **kwargs):
+            res = func(*args, **kwargs)
+            attempt = 0
+            while not res and attempt < times:
+                sleep(delay_secs)
+                res = func(*args, **kwargs)
+                if res:
+                    break
+                attempt += 1
+            return res
+        return _wrapped
+
+    return retry_decorator

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -39,6 +39,7 @@ from charms.reactive import is_state
 from charms.reactive import when, when_any, when_not
 from charms.reactive.helpers import data_changed
 from charms.kubernetes.common import get_version
+from charms.kubernetes.common import retry
 from charms.kubernetes.flagmanager import FlagManager
 
 from charmhelpers.core import hookenv
@@ -368,6 +369,7 @@ def push_api_data(kube_api):
 @when('kubernetes-master.components.started')
 def configure_cdk_addons():
     ''' Configure CDK addons '''
+    remove_state('cdk-addons.configured')
     dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower()
     args = [
         'arch=' + arch(),
@@ -376,15 +378,31 @@ def configure_cdk_addons():
         'enable-dashboard=' + dbEnabled
     ]
     check_call(['snap', 'set', 'cdk-addons'] + args)
-    try:
-        check_call(['cdk-addons.apply'])
-    except CalledProcessError:
+    if not addons_ready():
         hookenv.status_set('waiting', 'Waiting to retry addon deployment')
         remove_state('cdk-addons.configured')
         return
+
     set_state('cdk-addons.configured')
 
 
+@retry(times=3, delay_secs=20)
+def addons_ready():
+    """
+    Test if the add ons got installed
+
+    Returns: True is the addons got applied
+
+    """
+    try:
+        check_call(['cdk-addons.apply'])
+        return True
+    except CalledProcessError:
+        hookenv.log("Addons are not ready yet.")
+        return False
+
+
+
 @when('loadbalancer.available', 'certificates.ca.available',
       'certificates.client.cert.available')
 def loadbalancer_kubeconfig(loadbalancer, ca, client):
@@ -792,6 +810,7 @@ def setup_tokens(token, username, user):
         stream.write('{0},{1},{2}'.format(token, username, user))
 
 
+@retry(times=2, delay_secs=20)
 def all_kube_system_pods_running():
     ''' Check pod status in the kube-system namespace. Returns True if all
     pods are running, False otherwise. '''