diff --git a/API.md b/API.md index 9300cb6..63a2efa 100644 --- a/API.md +++ b/API.md @@ -43,13 +43,26 @@ * [get_marathon_tasks()](#get_marathon_tasks) * [service_healthy()](#service_healthy) * [wait_for_service_endpoint()](#wait_for_service_endpoint) + * [wait_for_service_endpoint_removal()](#wait_for_service_endpoint_removal) + * Spinner + * [wait_for()](#wait_for) + * [time_wait()](#time_wait) + * [elapse_time()](#elapse_time) * Tasks * [get_task()](#get_task) * [get_tasks()](#get_tasks) * [get_active_tasks()](#get_active_tasks) * [task_completed()](#task_completed) + * [wait_for_task()](#wait_for_task) + * [wait_for_task_property()](#wait_for_task_property) + * [wait_for_task_property_value()](#wait_for_task_property_value) + * [wait_for_dns()](#wait_for_dns) * ZooKeeper * [delete_zk_node()](#delete_zk_node) + * Marathon + * [deployment_wait()](#deployment_wait) + * [delete_all_apps()](#delete_all_apps) + * [delete_all_apps_wait()](#delete_all_apps_wait) * Masters * [partition_master()](#partition_master) * [reconnect_master()](#reconnect_master) @@ -680,6 +693,105 @@ timeout_sec | how long in seconds to wait before timing out | int | `120` wait_for_service_endpoint("marathon-user") ``` +### wait_for_service_endpoint_removal() + +Checks the service url returns HTTP 500 within a timeout if available it returns true on expiration it returns time to remove. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +**service_name** | the name of the service | str +timeout_sec | how long in seconds to wait before timing out | int | `120` + +##### *example usage* + +```python +# will wait +wait_for_service_endpoint_removal("marathon-user") +``` + +### wait_for() + +Waits for a function to return true or times out. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +**predicate** | the predicate function| fn +timeout_seconds | how long in seconds to wait before timing out | int | `120` +sleep_seconds | time to sleep between multiple calls to predicate | int | `1` +ignore_exceptions | ignore exceptions thrown by predicate | bool | True +inverse_predicate | if True look for False from predicate | bool | False + +##### *example usage* + +```python +# simple predicate +def deployment_predicate(client=None): + ... + +wait_for(deployment_predicate, timeout) + +# predicate with a parameter +def service_available_predicate(service_name): + ... + +wait_for(lambda: service_available_predicate(service_name), timeout_seconds=timeout_sec) + +``` + +### time_wait() + +Waits for a function to return true or times out. Returns the elapsed time of wait. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +**predicate** | the predicate function| fn +timeout_seconds | how long in seconds to wait before timing out | int | `120` +sleep_seconds | time to sleep between multiple calls to predicate | int | `1` +ignore_exceptions | ignore exceptions thrown by predicate | bool | True +inverse_predicate | if True look for False from predicate | bool | False + +##### *example usage* + +```python +# simple predicate +def deployment_predicate(client=None): + ... + +time_wait(deployment_predicate, timeout) + +# predicate with a parameter +def service_available_predicate(service_name): + ... + +time_wait(lambda: service_available_predicate(service_name), timeout_seconds=timeout_sec) + +``` + +### elapse_time() + +returns the time difference with a given precision. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +**start** | the start time | time +end | end time, if not provided current time is used | time | None +precision | the number decimal places to maintain | int | `3` + +##### *example usage* + +```python +# will wait +elapse_time("marathon-user") +``` + ### get_task() Get information about a task. @@ -728,7 +840,6 @@ for task in tasks: print("{} has state {}".format(task['id'], task['state'])) ``` - ### task_completed() Check whether a task has completed. @@ -748,6 +859,82 @@ while not task_completed('driver-20160517222552-0072'): time.sleep(5) ``` +### wait_for_task() + +Wait for a task to be reported running by Mesos. Returns the elapsed time of wait. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +service | framework service name | str +task | task name | str +timeout_sec | timeout | int | `120` + + +##### *example usage* + +```python +wait_for_task('marathon', 'marathon-user') +``` + +### wait_for_task_property() + +Wait for a task to be report having a specific property. Returns the elapsed time of wait. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +service | framework service name | str +task | task name | str +prop | property name | str +timeout_sec | timeout | int | `120` + + +##### *example usage* + +```python +wait_for_task_property('marathon', 'chronos', 'resources') +``` + +### wait_for_task_property_value() + +Wait for a task to be reported having a property with a specific value. Returns the elapsed time of wait. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +service | framework service name | str +task | task name | str +prop | property name | str +value | value of property | str +timeout_sec | timeout | int | `120` + + +##### *example usage* + +```python +wait_for_task_property_value('marathon', 'marathon-user', 'state', 'TASK_RUNNING') +``` + +### wait_for_dns() + +Wait for a task dns. Returns the elapsed time of wait. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +name | dns name | str +timeout_sec | timeout | int | `120` + +##### *example usage* + +```python +wait_for_dns('marathon-user.marathon.mesos') +``` ### delete_zk_node() @@ -766,6 +953,50 @@ node_name | the name of the node | str delete_zk_node('universe/marathon-user') ``` +### deployment_wait() + +Waits for Marathon Deployment to complete or times out. + +##### *parameters* + +parameter | description | type | default +--------- | ----------- | ---- | ------- +timeout | max time to wait for deployment | int | 120 + +##### *example usage* + +```python +# assuming a client.add_app() or similar +deployment_wait() +``` + +### delete_all_apps() + +Deletes all apps running on Marathon. + +##### *parameters* + +None. + +##### *example usage* + +```python +delete_all_apps() +``` + +### delete_all_apps_wait() + +Deletes all apps running on Marathon and waits for deployment to finish. + +##### *parameters* + +None. + +##### *example usage* + +```python +delete_all_apps_wait() +``` ### partition_master() diff --git a/shakedown/__init__.py b/shakedown/__init__.py index 2064415..543afe6 100644 --- a/shakedown/__init__.py +++ b/shakedown/__init__.py @@ -4,9 +4,11 @@ from shakedown.dcos.config import * from shakedown.dcos.command import * from shakedown.dcos.file import * +from shakedown.dcos.marathon import * from shakedown.dcos.network import * from shakedown.dcos.package import * from shakedown.dcos.service import * +from shakedown.dcos.spinner import * from shakedown.dcos.task import * from shakedown.dcos.zookeeper import * from shakedown.dcos.agent import * diff --git a/shakedown/dcos/marathon.py b/shakedown/dcos/marathon.py new file mode 100644 index 0000000..8ede529 --- /dev/null +++ b/shakedown/dcos/marathon.py @@ -0,0 +1,21 @@ +from dcos import marathon +from shakedown.dcos.spinner import * + + +def deployment_predicate(): + client = marathon.create_client() + return len(client.get_deployments()) == 0 + + +def deployment_wait(timeout=120): + time_wait(deployment_predicate, timeout) + + +def delete_all_apps(): + client = marathon.create_client() + client.remove_group("/") + + +def delete_all_apps_wait(): + delete_all_apps() + deployment_wait() diff --git a/shakedown/dcos/package.py b/shakedown/dcos/package.py index 471d734..52f1ed3 100644 --- a/shakedown/dcos/package.py +++ b/shakedown/dcos/package.py @@ -231,7 +231,6 @@ def get_package_repos( return cosmos.get_repos() - def add_package_repo( repo_name, repo_url, diff --git a/shakedown/dcos/service.py b/shakedown/dcos/service.py index 9e02931..ef6fcdb 100644 --- a/shakedown/dcos/service.py +++ b/shakedown/dcos/service.py @@ -1,4 +1,5 @@ from dcos import (marathon, mesos) +from shakedown.dcos.spinner import * def get_service( @@ -159,6 +160,7 @@ def get_service_ips( return ips + def service_healthy(service_name, app_id=None): """ Check whether a named service is healthy @@ -187,26 +189,36 @@ def service_healthy(service_name, app_id=None): return False -def wait_for_service_endpoint(service,timeout_sec=120): - """Checks the service url returns HTTP 200 within a timeout if available it returns true on expiration it returns false""" - - url = dcos_service_url(service) - now = time.time() - future = now + timeout_sec - time.sleep(5) +def service_available_predicate(service_name): + url = dcos_service_url(service_name) + try: + response = http.get(url) + return response.status_code == 200 + except Exception as e: + return False - while now < future: - response = None - try: - response = http.get(url) - except Exception as e: - pass - if response is None: - time.sleep(5) - now = time.time() - elif response.status_code == 200: +def service_unavailable_predicate(service_name): + url = dcos_service_url(service_name) + try: + response = http.get(url) + except DCOSHTTPException as e: + if e.response.status_code == 500: return True + else: + return False - return False + +def wait_for_service_endpoint(service_name, timeout_sec=120): + """Checks the service url if available it returns true, on expiration + it returns false""" + + return time_wait(lambda: service_available_predicate(service_name), timeout_seconds=timeout_sec) + + +def wait_for_service_endpoint_removal(service_name, timeout_sec=120): + """Checks the service url if it is removed it returns true, on expiration + it returns false""" + + return time_wait(lambda: service_unavailable_predicate(service_name)) diff --git a/shakedown/dcos/spinner.py b/shakedown/dcos/spinner.py new file mode 100644 index 0000000..e081d53 --- /dev/null +++ b/shakedown/dcos/spinner.py @@ -0,0 +1,86 @@ +import time as time_module + + +def wait_for(predicate, timeout_seconds=120, sleep_seconds=1, ignore_exceptions=True, inverse_predicate=False): + """ waits or spins for a predicate. Predicate is in function that returns a True or False. + An exception in the function will be returned. + A timeout will throw a TimeoutExpired Exception. + + """ + + timeout = Deadline.create_deadline(timeout_seconds) + while True: + try: + result = predicate() + except Exception as e: + if not ignore_exceptions: + raise e + else: + if (not inverse_predicate and result) or (inverse_predicate and not result): + return + if timeout.is_expired(): + raise TimeoutExpired(timeout_seconds, str(predicate)) + time_module.sleep(sleep_seconds) + + +def time_wait(predicate, timeout_seconds=120, sleep_seconds=1, ignore_exceptions=True, inverse_predicate=False): + """ waits or spins for a predicate and returns the time of the wait. + An exception in the function will be returned. + A timeout will throw a TimeoutExpired Exception. + + """ + start = time_module.time() + wait_for(predicate, timeout_seconds, sleep_seconds, ignore_exceptions, inverse_predicate) + return elapse_time(start) + + +def elapse_time(start, end=None, precision=3): + """ Simple time calculation utility. Given a start time, it will provide an elapse time. + """ + if end is None: + end = time_module.time() + return round(end-start, precision) + + +class Deadline(object): + + def is_expired(self): + raise NotImplementedError() + + @staticmethod + def create_deadline(seconds): + if seconds is None: + return Forever() + return Within(seconds) + + +class Within(Deadline): + + def __init__(self, seconds): + super(Within, self).__init__() + self._deadline = time_module.time() + seconds + + def is_expired(self): + return time_module.time() >= self._deadline + + +class Forever(Deadline): + + def is_expired(self): + return False + + +class TimeoutExpired(Exception): + def __init__(self, timeout_seconds, what): + super(TimeoutExpired, self).__init__(timeout_seconds, what) + self._timeout_seconds = timeout_seconds + self._what = what + + def __str__(self): + return "Timeout of {0} seconds expired waiting for {1}".format(self._timeout_seconds, self._what) + + def __repr__(self): + return "{0}: {1}".format(type(self).__name__, self) + + def __unicode__(self): + return u"Timeout of {0} seconds expired waiting for {1}".format(self._timeout_seconds, self._what) diff --git a/shakedown/dcos/task.py b/shakedown/dcos/task.py index 95da912..58cce0c 100644 --- a/shakedown/dcos/task.py +++ b/shakedown/dcos/task.py @@ -1,6 +1,9 @@ from dcos import mesos from shakedown.dcos.helpers import * +from shakedown.dcos.service import * +from shakedown.dcos.spinner import * +from shakedown.dcos import * import shakedown import time @@ -62,6 +65,7 @@ def task_completed(task_id): return False + def wait_for_task_completion(task_id): """ Block until the task completes @@ -72,3 +76,50 @@ def wait_for_task_completion(task_id): """ while not task_completed(task_id): time.sleep(1) + + +def task_property_value_predicate(service, task, prop, value): + try: + response = get_service_task(service, task) + except Exception as e: + pass + + return (response is not None) and (response[prop] == value) + + +def task_predicate(service, task): + return task_property_value_predicate(service, task, 'state', 'TASK_RUNNING') + + +def task_property_present_predicate(service, task, prop): + """ True if the json_element passed is present for the task specified. + """ + try: + response = get_service_task(service, task) + except Exception as e: + pass + + return (response is not None) and (prop in response) + + +def wait_for_task(service, task, timeout_sec=120): + """Waits for a task which was launched to be launched""" + return time_wait(lambda: task_predicate(service, task), timeout_seconds=timeout_sec) + + +def wait_for_task_property(service, task, prop, timeout_sec=120): + """Waits for a task which was launched to be launched""" + return time_wait(lambda: task_property_present_predicate(service, task, prop), timeout_seconds=timeout_sec) + + +def wait_for_task_property_value(service, task, prop, value, timeout_sec=120): + return time_wait(lambda: task_property_value_predicate(service, task, prop, value), timeout_seconds=timeout_sec) + + +def dns_predicate(name): + dns = dcos_dns_lookup(name) + return dns[0].get('ip') is not None + + +def wait_for_dns(name, timeout_sec=120): + return time_wait(lambda: dns_predicate(name), timeout_seconds=timeout_sec) diff --git a/tests/acceptance/test_dcos_package.py b/tests/acceptance/test_dcos_package.py index f5e8e8c..b3c978f 100644 --- a/tests/acceptance/test_dcos_package.py +++ b/tests/acceptance/test_dcos_package.py @@ -13,8 +13,18 @@ def test_uninstall_package_and_wait(): uninstall_package_and_wait('chronos') assert package_installed('chronos') == False +def task_cpu_predicate(service, task): + try: + response = get_service_task(service, task) + except Exception as e: + pass + + return (response is not None) and ('resources' in response) and ('cpus' in response['resources']) + + def test_install_package_with_json_options(): install_package_and_wait('chronos', None, 'big-chronos', None, {"chronos": {"cpus": 2}}) + wait_for(lambda: task_cpu_predicate('marathon', 'big-chronos')) assert get_service_task('marathon', 'big-chronos')['resources']['cpus'] == 2 uninstall_package_and_wait('chronos')