Commit 067216fc authored by Alexander Efremkin's avatar Alexander Efremkin
Browse files

Do not fail poweron job on verification error

A vexing failure of poweron job caused by verification mismatch is now opt-in with a CLI flag "--verify", with default being no verification.

It is more common that not that system state is slightly off compared to expected in the tessia database; such discrepancies are still logged in job output, but do not produce an error unless a verification flag is set.

DCO 1.1 Signed-off-by: Alexander Efremkin <aefr@de.ibm.com>
parent c3d27bc4
Pipeline #244624661 passed with stages
in 38 minutes and 55 seconds
......@@ -340,7 +340,9 @@ def poweroff(ctx, name, verbosity, bg_flag):
@click.option('--force', is_flag=True,
help="force a poweron even if system is already up")
@click.option('--noverify', is_flag=True,
help="do not perform any system state verification")
help="Skip system state verification [deprecated, default]")
@click.option('--verify', is_flag=True,
help="Perform system state verification")
@click.option(
'--exclusive', is_flag=True,
help="stop ALL other systems under same hypervisor, USE WITH CARE!")
......@@ -358,9 +360,12 @@ def poweron(ctx, name, **kwargs):
fetch_item(client.Systems, {'name': name},
'system {} not found.'.format(name))
req_params = {'systems': [
{'action': 'poweron', 'name': name, 'profile_override': {}}
]}
req_params = {
'systems': [
{'action': 'poweron', 'name': name, 'profile_override': {}}
],
'verify': kwargs['verify']
}
# profile specified: like system, make sure it exists first
if kwargs['profile']:
fetch_item(
......@@ -370,7 +375,7 @@ def poweron(ctx, name, **kwargs):
req_params['systems'][0]['profile'] = kwargs['profile']
if kwargs['noverify']:
req_params['verify'] = False
click.echo('"noverify" is now default and can be omitted')
if kwargs['exclusive']:
req_params['systems'][0]['action'] = 'poweron-exclusive'
if kwargs['force']:
......
......@@ -1359,12 +1359,14 @@ tasks:
- 'no such option: --forc Did you mean --force?'
- - system poweron --name=cpc3lp52 --force
- 'Failed to establish a new connection'
- - system poweron --name=cpc3lp52 --noverify=true
- '--noverify option does not take a value'
- - system poweron --name=cpc3lp52 --noverif
- 'no such option: --noverif Did you mean --noverify?'
- - system poweron --name=cpc3lp52 --noverify
- - system poweron --name=cpc3lp52 --verify=true
- '--verify option does not take a value'
- - system poweron --name=cpc3lp52 --verif
- 'no such option: --verif Did you mean --verify?'
- - system poweron --name=cpc3lp52 --verify
- 'Failed to establish a new connection'
- - system poweron --name=cpc3lp52 --noverify
- 'is now default and can be omitted'
- - system poweron --name=cpc3lp52 --exclusive=true
- '--exclusive option does not take a value'
- - system poweron --name=cpc3lp52 --exclus
......
......@@ -128,6 +128,9 @@ class PostInstallChecker:
# fetched at verification time
self._facts = None
# collected mismatches
self._mismatches = []
# __init__()
def _exec_ansible(self, module, args=None):
......@@ -876,6 +879,7 @@ class PostInstallChecker:
misconf_exc = Misconfiguration(*args, **kwargs)
if self._permissive:
self._logger.warning(str(misconf_exc))
self._mismatches.append(misconf_exc)
return
raise misconf_exc
# _report()
......@@ -1158,11 +1162,15 @@ class PostInstallChecker:
areas (list): areas to check, possible values are cpu, kernel, os,
memory, network, storage.
Returns:
list: list of mismatches
Raises:
ValueError: in case an invalid area is specified
Misconfiguration: if an actual parameter does not match the
expected value
"""
self._mismatches = []
self._fetch_facts()
self._logger.debug(
'expected params are: %s', pformat(self._expected_params))
......@@ -1185,5 +1193,6 @@ class PostInstallChecker:
except KeyError:
raise ValueError('Invalid area {}'.format(area))
area_method()
return self._mismatches
# verify()
# PostInstallChecker
......@@ -110,11 +110,14 @@ REQUEST_SCHEMA = {
#
# CODE
#
class PowerManagerMachine(BaseMachine):
"""
This machine is responsible for performing power management actions
(poweron, poweroff) of the managed systems.
"""
def __init__(self, params):
"""
See base class docstring.
......@@ -939,7 +942,10 @@ class PowerManagerMachine(BaseMachine):
raise TimeoutError(
'Could not establish a connection to system {} after {} '
'seconds'.format(system_name, LOAD_TIMEOUT))
if not self._state_match(profile_obj):
# do a state match always to log incinsitencies,
# raise only if requested
if (not self._state_match(profile_obj) and
self._params.get('verify')):
raise RuntimeError(
'Failed to poweron system {} with expected configuration'
.format(system_name))
......@@ -976,22 +982,22 @@ class PowerManagerMachine(BaseMachine):
'supported', system_name)
return True
verify_flag = self._params.get('verify', True)
if not verify_flag:
if not self._params.get('verify'):
self._logger.info(
'Potential configuration mismatches will be reported as '
'warnings because verify flag is off')
try:
checker = post_install.PostInstallChecker(
system_prof, permissive=not verify_flag)
checker.verify()
system_prof, permissive=True)
# checked may still throw on unexpected errors
mismatches = checker.verify()
except Exception as exc:
if verify_flag:
self._logger.warning('State verification of system %s '
'failed: %s', system_name, str(exc))
return False
self._logger.warning('State verification of system %s '
'failed: %s', system_name, str(exc))
return False
return True
return not mismatches
# _state_match()
def cleanup(self):
......
......@@ -98,6 +98,7 @@ class TestPowerManagerMachine(TestCase):
machine.post_install, 'PostInstallChecker', autospec=True)
self._mock_post_cls = patcher.start()
self._mock_post_obj = self._mock_post_cls.return_value
self._mock_post_obj.verify.return_value = []
self.addCleanup(patcher.stop)
# setUp()
......@@ -439,7 +440,7 @@ class TestPowerManagerMachine(TestCase):
# validate stage verify
self._assert_system_up_action(prof_obj, 1)
self._mock_post_cls.assert_called_with(prof_obj, permissive=False)
self._mock_post_cls.assert_called_with(prof_obj, permissive=True)
self._mock_post_obj.verify.assert_called_with()
# test_multiple_actions()
......@@ -622,7 +623,8 @@ class TestPowerManagerMachine(TestCase):
'name': system_obj.name,
'force': True
},
]
],
'verify': True,
})
prof_obj = self._get_profile(system_obj.name)
vol_obj = prof_obj.storage_volumes_rel[0]
......@@ -855,7 +857,7 @@ class TestPowerManagerMachine(TestCase):
self._assert_system_up_action(prof_obj, 1)
self.assertEqual(
self._mock_post_cls.call_args_list[0],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[0], call())
......@@ -925,7 +927,7 @@ class TestPowerManagerMachine(TestCase):
# validate call to verify if hypervisor matches profile
self.assertEqual(
self._mock_post_cls.call_args_list[0],
call(hyp_prof, permissive=False))
call(hyp_prof, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[0], call())
......@@ -943,7 +945,7 @@ class TestPowerManagerMachine(TestCase):
self._assert_system_up_action(prof_obj, 2)
self.assertEqual(
self._mock_post_cls.call_args_list[1],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[1], call())
......@@ -1024,7 +1026,7 @@ class TestPowerManagerMachine(TestCase):
# validate call to verify if guest state matched profile
self.assertEqual(
self._mock_post_cls.call_args_list[0],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[0], call())
......@@ -1038,7 +1040,7 @@ class TestPowerManagerMachine(TestCase):
# validate call to verify if guest state matched profile
self.assertEqual(
self._mock_post_cls.call_args_list[1],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[1], call())
......@@ -1142,7 +1144,7 @@ class TestPowerManagerMachine(TestCase):
self._assert_system_up_action(hyp_prof, 0)
# validate call to verify hypervisor profile
self._mock_post_cls.assert_called_with(hyp_prof, permissive=False)
self._mock_post_cls.assert_called_with(hyp_prof, permissive=True)
self._mock_post_obj.verify.assert_called_with()
# no call to power on system
......@@ -1169,7 +1171,8 @@ class TestPowerManagerMachine(TestCase):
'action': 'poweron',
'name': system_obj.name,
},
]
],
'verify': True,
})
error_msg = (
'Failed to poweron system {} with expected configuration'.format(
......@@ -1185,7 +1188,7 @@ class TestPowerManagerMachine(TestCase):
self._assert_poweron_action('hmc', hyp_prof, prof_obj, 0)
# validate stage verify
self._mock_post_cls.assert_called_with(prof_obj, permissive=False)
self._mock_post_cls.assert_called_with(prof_obj, permissive=True)
self._mock_post_obj.verify.assert_called_with()
# test_poweron_verify_fails()
......@@ -1233,7 +1236,7 @@ class TestPowerManagerMachine(TestCase):
# validate that post install was called to verify profile
self.assertEqual(
self._mock_post_cls.call_args_list[0],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[0], call())
# test_poweron_zvm_dasd()
......@@ -1282,7 +1285,7 @@ class TestPowerManagerMachine(TestCase):
# validate that post install was called to verify profile
self.assertEqual(
self._mock_post_cls.call_args_list[0],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[0], call())
# test_poweron_zvm()
......@@ -1322,7 +1325,7 @@ class TestPowerManagerMachine(TestCase):
# test_poweron_zvm_missing_cred()
def test_verify_off(self):
def test_verify_on(self):
"""
Try a poweron with verify flag off. This test also covers the usage
of the default profile when none was specified.
......@@ -1352,7 +1355,7 @@ class TestPowerManagerMachine(TestCase):
'name': system_obj.name,
},
],
'verify': False
'verify': True
})
machine_obj = machine.PowerManagerMachine(request)
......@@ -1430,7 +1433,7 @@ class TestPowerManagerMachine(TestCase):
# validate that post install was called to verify profile
self.assertEqual(
self._mock_post_cls.call_args_list[0],
call(prof_obj, permissive=False))
call(prof_obj, permissive=True))
self.assertEqual(
self._mock_post_obj.verify.call_args_list[0], call())
# test_poweron_zvm_modified_date()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment