[389-commits] [389-ds-base] branch 389-ds-base-1.4.0 updated: Ticket 50327 - Add replication conflict entry support to lib389/CLI

This is an automated email from the git hooks/post-receive script. mreynolds pushed a commit to branch 389-ds-base-1.4.0 in repository 389-ds-base. The following commit(s) were added to refs/heads/389-ds-base-1.4.0 by this push: new fec59e0 Ticket 50327 - Add replication conflict entry support to lib389/CLI fec59e0 is described below commit fec59e010010b30fd0e04f148b0066204eb65251 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; AuthorDate: Tue Apr 16 15:23:16 2019 -0400 Ticket 50327 - Add replication conflict entry support to lib389/CLI Description: Added Conflict Entry and Glue entry classes to lib389, and updated dsconf to allow for conflict entry management. Made some other minor changes to mapped objects: - Added an attribute list option to display() - Added a recursive delete option to delete() https://pagure.io/389-ds-base/issue/50327 Reviewed by: firstyear, lkrispen, and spichugi(Thanks!!!) (cherry picked from commit 4f7c05e2879cee7d205531edb64b19ad799e20bd) --- src/lib389/cli/dsconf | 2 + src/lib389/lib389/_mapped_object.py | 18 ++- src/lib389/lib389/cli_conf/conflicts.py | 127 +++++++++++++++ src/lib389/lib389/cli_conf/monitor.py | 1 + src/lib389/lib389/conflicts.py | 175 +++++++++++++++++++++ src/lib389/lib389/tests/cli/conf_conflicts_test.py | 161 +++++++++++++++++++ 6 files changed, 476 insertions(+), 8 deletions(-) diff --git a/src/lib389/cli/dsconf b/src/lib389/cli/dsconf index 37e6282..9f747c0 100755 --- a/src/lib389/cli/dsconf +++ b/src/lib389/cli/dsconf @@ -30,6 +30,7 @@ from lib389.cli_conf import pwpolicy as cli_pwpolicy from lib389.cli_conf import backup as cli_backup from lib389.cli_conf import replication as cli_replication from lib389.cli_conf import chaining as cli_chaining +from lib389.cli_conf import conflicts as cli_repl_conflicts from lib389.cli_base import disconnect_instance, connect_instance from lib389.cli_base.dsrc import dsrc_to_ldap, dsrc_arg_concat from lib389.cli_base import setup_script_logger @@ -85,6 +86,7 @@ cli_pwpolicy.create_parser(subparsers) cli_replication.create_parser(subparsers) cli_sasl.create_parser(subparsers) cli_schema.create_parser(subparsers) +cli_repl_conflicts.create_parser(subparsers) argcomplete.autocomplete(parser) diff --git a/src/lib389/lib389/_mapped_object.py b/src/lib389/lib389/_mapped_object.py index a141877..9486979 100644 --- a/src/lib389/lib389/_mapped_object.py +++ b/src/lib389/lib389/_mapped_object.py @@ -1,5 +1,5 @@ # --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2016 Red Hat, Inc. +# Copyright (C) 2019 Red Hat, Inc. # Copyright (C) 2019 William Brown <william(a)blackhats.net.au&gt; # All rights reserved. # @@ -142,12 +142,11 @@ class DSLdapObject(DSLogging): return True - def display(self): + def display(self, attrlist=['*']): """Get an entry but represent it as a string LDIF :returns: LDIF formatted string """ - e = self._instance.search_ext_s(self._dn, ldap.SCOPE_BASE, self._object_filter, attrlist=["*"], serverctrls=self._server_controls, clientctrls=self._client_controls)[0] return e.__repr__() @@ -464,7 +463,8 @@ class DSLdapObject(DSLogging): all_attrs_dict = self.get_all_attrs() # removing _compate_exclude attrs from all attrs compare_attrs = set(all_attrs_dict.keys()) - set(self._compare_exclude) - compare_attrs_dict = {attr:all_attrs_dict[attr] for attr in compare_attrs} + compare_attrs_dict = {attr: all_attrs_dict[attr] for attr in compare_attrs} + return compare_attrs_dict def get_all_attrs(self, use_json=False): @@ -495,7 +495,9 @@ class DSLdapObject(DSLogging): self._log.debug("%s get_attrs_vals_utf8(%r)" % (self._dn, keys)) if self._instance.state != DIRSRV_STATE_ONLINE: raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE") - entry = self._instance.search_ext_s(self._dn, ldap.SCOPE_BASE, self._object_filter, attrlist=keys, serverctrls=self._server_controls, clientctrls=self._client_controls, escapehatch='i am sure')[0] + entry = self._instance.search_ext_s(self._dn, ldap.SCOPE_BASE, self._object_filter, attrlist=keys, + serverctrls=self._server_controls, clientctrls=self._client_controls, + escapehatch='i am sure')[0] vset = entry.getValuesSet(keys) r = {} for (k, vo) in vset.items(): @@ -637,7 +639,7 @@ class DSLdapObject(DSLogging): pass # Modifies the DN of an entry to the new fqdn provided - def rename(self, new_rdn, newsuperior=None): + def rename(self, new_rdn, newsuperior=None, deloldrdn=True): """Renames the object within the tree. If you provide a newsuperior, this will move the object in the tree. @@ -660,7 +662,7 @@ class DSLdapObject(DSLogging): return self._instance.rename_s(self._dn, new_rdn, newsuperior, serverctrls=self._server_controls, clientctrls=self._client_controls) search_base = self._basedn - if newsuperior != None: + if newsuperior is not None: # Well, the new DN should be rdn + newsuperior. self._dn = '%s,%s' % (new_rdn, newsuperior) else: @@ -672,7 +674,7 @@ class DSLdapObject(DSLogging): # assert we actually got the change right .... - def delete(self): + def delete(self, recursive=False): """Deletes the object defined by self._dn. This can be changed with the self._protected flag! """ diff --git a/src/lib389/lib389/cli_conf/conflicts.py b/src/lib389/lib389/cli_conf/conflicts.py new file mode 100644 index 0000000..620f68c --- /dev/null +++ b/src/lib389/lib389/cli_conf/conflicts.py @@ -0,0 +1,127 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2019 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +import json +from lib389.conflicts import (ConflictEntries, ConflictEntry, GlueEntries, GlueEntry) + +conflict_attrs = ['nsds5replconflict', '*'] + + +def list_conflicts(inst, basedn, log, args): + conflicts = ConflictEntries(inst, args.suffix).list() + if args.json: + results = [] + for conflict in conflicts: + results.append(json.loads(conflict.get_all_attrs_json())) + log.info(json.dumps({'type': 'list', 'items': results})) + else: + if len(conflicts) > 0: + for conflict in conflicts: + log.info(conflict.display(conflict_attrs)) + else: + log.info("There were no conflict entries found under the suffix") + + +def cmp_conflict(inst, basedn, log, args): + conflict = ConflictEntry(inst, args.DN) + valid_entry = conflict.get_valid_entry() + + if args.json: + results = [] + results.append(json.loads(conflict.get_all_attrs_json())) + results.append(json.loads(valid_entry.get_all_attrs_json())) + log.info(json.dumps({'type': 'list', 'items': results})) + else: + log.info("Conflict Entry:\n") + log.info(conflict.display(conflict_attrs)) + log.info("Valid Entry:\n") + log.info(valid_entry.display(conflict_attrs)) + + +def del_conflict(inst, basedn, log, args): + conflict = ConflictEntry(inst, args.DN) + conflict.delete() + + +def swap_conflict(inst, basedn, log, args): + conflict = ConflictEntry(inst, args.DN) + conflict.swap() + + +def convert_conflict(inst, basedn, log, args): + conflict = ConflictEntry(inst, args.DN) + conflict.convert(args.new_rdn) + + +def list_glue(inst, basedn, log, args): + glues = GlueEntries(inst, args.suffix).list() + if args.json: + results = [] + for glue in glues: + results.append(json.loads(glue.get_all_attrs_json())) + log.info(json.dumps({'type': 'list', 'items': results})) + else: + if len(glues) > 0: + for glue in glues: + log.info(glue.display(conflict_attrs)) + else: + log.info("There were no glue entries found under the suffix") + + +def del_glue(inst, basedn, log, args): + glue = GlueEntry(inst, args.DN) + glue.delete_all() + + +def convert_glue(inst, basedn, log, args): + glue = GlueEntry(inst, args.DN) + glue.convert() + + +def create_parser(subparsers): + conflict_parser = subparsers.add_parser('repl-conflict', help="Manage replication conflicts") + subcommands = conflict_parser.add_subparsers(help='action') + + # coinflict entry arguments + list_parser = subcommands.add_parser('list', help="List conflict entries") + list_parser.add_argument('suffix', help='The backend name, or suffix, to look for conflict entries') + list_parser.set_defaults(func=list_conflicts) + + cmp_parser = subcommands.add_parser('compare', help="Compare the conflict entry with its valid counterpart") + cmp_parser.add_argument('DN', help='The DN of the conflict entry') + cmp_parser.set_defaults(func=cmp_conflict) + + del_parser = subcommands.add_parser('delete', help="Delete a conflict entry") + del_parser.add_argument('DN', help='The DN of the conflict entry') + del_parser.set_defaults(func=del_conflict) + + replace_parser = subcommands.add_parser('swap', help="Replace the valid entry with the conflict entry") + replace_parser.add_argument('DN', help='The DN of the conflict entry') + replace_parser.set_defaults(func=swap_conflict) + + replace_parser = subcommands.add_parser('convert', help="Convert the conflict entry to a valid entry, " + "while keeping the original valid entry counterpart. " + "This requires that the converted conflict entry have " + "a new RDN value. For example: \"cn=my_new_rdn_value\".") + replace_parser.add_argument('DN', help='The DN of the conflict entry') + replace_parser.add_argument('--new-rdn', required=True, help="The new RDN for the converted conflict entry. " + "For example: \"cn=my_new_rdn_value\"") + replace_parser.set_defaults(func=convert_conflict) + + # Glue entry arguments + list_glue_parser = subcommands.add_parser('list-glue', help="List replication glue entries") + list_glue_parser.add_argument('suffix', help='The backend name, or suffix, to look for glue entries') + list_glue_parser.set_defaults(func=list_glue) + + del_glue_parser = subcommands.add_parser('delete-glue', help="Delete the glue entry and its child entries") + del_glue_parser.add_argument('DN', help='The DN of the glue entry') + del_glue_parser.set_defaults(func=del_glue) + + convert_glue_parser = subcommands.add_parser('convert-glue', help="Convert the glue entry into a regular entry") + convert_glue_parser.add_argument('DN', help='The DN of the glue entry') + convert_glue_parser.set_defaults(func=convert_glue) diff --git a/src/lib389/lib389/cli_conf/monitor.py b/src/lib389/lib389/cli_conf/monitor.py index a704bea..53637e1 100644 --- a/src/lib389/lib389/cli_conf/monitor.py +++ b/src/lib389/lib389/cli_conf/monitor.py @@ -1,5 +1,6 @@ # --- BEGIN COPYRIGHT BLOCK --- # Copyright (C) 2019 William Brown <william(a)blackhats.net.au&gt; +# Copyright (C) 2019 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). diff --git a/src/lib389/lib389/conflicts.py b/src/lib389/lib389/conflicts.py new file mode 100644 index 0000000..b1f86e0 --- /dev/null +++ b/src/lib389/lib389/conflicts.py @@ -0,0 +1,175 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2019 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- +# + +import ldap +from lib389._mapped_object import DSLdapObject, DSLdapObjects, _gen_filter + + +class ConflictEntry(DSLdapObject): + """A replication conflict entry + + :param instance: An instance + :type instance: lib389.DirSrv + :param dn: The DN of the conflict entry + :type dn: str + """ + def __init__(self, instance, dn=None): + super(ConflictEntry, self).__init__(instance, dn) + self._rdn_attribute = 'cn' + self._create_objectclasses = ['ldapsubentry'] + self._protected = False + self._object_filter = '(objectclass=ldapsubentry)' + + def convert(self, new_rdn): + """Convert conflict entry to a vlid entry, but we need to + give the conflict entry a new rdn since we are not replacing + the existing valid counterpart entry. + """ + + # Get the conflict entry info + conflict_value = self.get_attr_val_utf8('nsds5ReplConflict') + entry_dn = conflict_value.split(' ', 3)[2] + entry_rdn = ldap.explode_dn(entry_dn, 1)[0] + rdn_attr = entry_dn.split('=', 1)[0] + + # Rename conflict entry + self.rename(new_rdn, deloldrdn=False) + + # Cleanup entry + self.remove(rdn_attr, entry_rdn) + if self.present('objectclass', 'ldapsubentry'): + self.remove('objectclass', 'ldapsubentry') + self.remove_all('nsds5ReplConflict') + + def swap(self): + """Make the conflict entry the real valid entry. Delete old valid entry, + and rename the conflict + """ + + # Get the conflict entry info + conflict_value = self.get_attr_val_utf8('nsds5ReplConflict') + entry_dn = conflict_value.split(' ', 3)[2] + entry_rdn = ldap.explode_dn(entry_dn, 1)[0] + + # Gather the RDN details + rdn_attr = entry_dn.split('=', 1)[0] + new_rdn = "{}={}".format(rdn_attr, entry_rdn) + tmp_rdn = new_rdn + 'tmp' + + # Delete valid entry (to be replaced by conflict entry) + original_entry = DSLdapObject(self._instance, dn=entry_dn) + original_entry._protected = False + original_entry.delete() + + # Rename conflict entry to tmp rdn so we can clean up the rdn attr + self.rename(tmp_rdn, deloldrdn=False) + + # Cleanup entry + self.remove(rdn_attr, entry_rdn) + if self.present('objectclass', 'ldapsubentry'): + self.remove('objectclass', 'ldapsubentry') + self.remove_all('nsds5ReplConflict') + + # Rename to the final/correct rdn + self.rename(new_rdn, deloldrdn=True) + + def get_valid_entry(self): + """Get the conflict entry's valid counterpart entry + """ + # Get the conflict entry info + conflict_value = self.get_attr_val_utf8('nsds5ReplConflict') + entry_dn = conflict_value.split(' ', 3)[2] + + # Get the valid entry + return DSLdapObject(self._instance, dn=entry_dn) + + +class ConflictEntries(DSLdapObjects): + """Represents the set of tombstone objects that may exist on + this replica. Tombstones are locally generated, so they are + unique to individual masters, and may or may not correlate + to tombstones on other masters. + + :param instance: An instance + :type instance: lib389.DirSrv + :param basedn: Tree to search for tombstones in + :type basedn: str + """ + def __init__(self, instance, basedn): + super(ConflictEntries, self).__init__(instance) + self._objectclasses = ['ldapsubentry'] + # Try some common ones .... + self._filterattrs = ['nsds5replconflict', 'objectclass'] + self._childobject = ConflictEntry + self._basedn = basedn + + def _get_objectclass_filter(self): + return "(&(objectclass=ldapsubentry)(nsds5replconflict=*))" + + +class GlueEntry(DSLdapObject): + """A replication glue entry + + :param instance: An instance + :type instance: lib389.DirSrv + :param dn: The DN of the conflict entry + :type dn: str + """ + def __init__(self, instance, dn=None): + super(GlueEntry, self).__init__(instance, dn) + self._rdn_attribute = '' + self._create_objectclasses = ['glue'] + self._protected = False + self._object_filter = '(objectclass=glue)' + + def convert(self): + """Convert entry into real entry + """ + self.remove_all('nsds5replconflict') + self.remove('objectclass', 'glue') + + def delete_all(self): + """Remove glue entry and its children. Depending on the situation the URP + mechanism can turn the parent glue entry into a tombstone before we get + a chance to delete it. This results in a NO_SUCH_OBJECT exception + """ + delete_count = 0 + filterstr = "(|(objectclass=*)(objectclass=ldapsubentry))" + ents = self._instance.search_s(self._dn, ldap.SCOPE_SUBTREE, filterstr, escapehatch='i am sure') + for ent in sorted(ents, key=lambda e: len(e.dn), reverse=True): + try: + self._instance.delete_ext_s(ent.dn, serverctrls=self._server_controls, clientctrls=self._client_controls, escapehatch='i am sure') + delete_count += 1 + except ldap.NO_SUCH_OBJECT as e: + if len(ents) > 0 and delete_count == (len(ents) - 1): + # This is the parent glue entry - it was removed by URP + pass + else: + raise e + + +class GlueEntries(DSLdapObjects): + """Represents the set of glue entries that may exist on + this replica. + + :param instance: An instance + :type instance: lib389.DirSrv + :param basedn: Tree to search for tombstones in + :type basedn: str + """ + def __init__(self, instance, basedn): + super(GlueEntries, self).__init__(instance) + self._objectclasses = ['glue'] + # Try some common ones .... + self._filterattrs = ['nsds5replconflict', 'objectclass'] + self._childobject = GlueEntry + self._basedn = basedn + + def _get_objectclass_filter(self): + return _gen_filter(['objectclass'], ['glue']) diff --git a/src/lib389/lib389/tests/cli/conf_conflicts_test.py b/src/lib389/lib389/tests/cli/conf_conflicts_test.py new file mode 100644 index 0000000..1624c28 --- /dev/null +++ b/src/lib389/lib389/tests/cli/conf_conflicts_test.py @@ -0,0 +1,161 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2018 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + + +import io +import sys +import pytest +import time +import json +from lib389.cli_base import LogCapture, FakeArgs +from lib389.utils import * +from lib389._constants import * +from lib389.idm.nscontainer import nsContainers +from lib389.topologies import topology_m2 as topo +from lib389.cli_conf.conflicts import (list_conflicts, cmp_conflict, del_conflict, swap_conflict, + convert_conflict, list_glue, del_glue, convert_glue) +from lib389.utils import ds_is_older +pytestmark = pytest.mark.skipif(ds_is_older('1.4.0'), reason="Not implemented") + + +def _create_container(inst, dn, name): + """Creates container entry""" + containers = nsContainers(inst, dn) + container = containers.create(properties={'cn': name}) + time.sleep(1) + return container + + +def _delete_container(container): + """Deletes container entry""" + container.delete() + time.sleep(1) + + +def test_conflict_cli(topo): + """Test manageing replication conflict entries + :id: 800f432a-52ab-4661-ac66-a2bdd9b984d8 + :setup: two masters + :steps: + 1. Create replication conflict entries + 2. List conflicts + 3. Compare conflict entry + 4. Delete conflict + 5. Resurrect conflict + 6. Swap conflict + 7. List glue entry + 8. Delete glue entry + 9. Convert glue entry + + :expectedresults: + 1. Success + 2. Success + 3. Success + 4. Success + 5. Success + 6. Success + 7. Success + 8. Success + 9. Success + 10. Success + """ + + # Setup our default parameters for CLI functions + topo.logcap = LogCapture() + sys.stdout = io.StringIO() + args = FakeArgs() + args.DN = "" + args.suffix = DEFAULT_SUFFIX + args.json = True + + m1 = topo.ms["master1"] + m2 = topo.ms["master2"] + + topo.pause_all_replicas() + + # Create entries + _create_container(m1, DEFAULT_SUFFIX, 'conflict_parent1') + _create_container(m2, DEFAULT_SUFFIX, 'conflict_parent1') + _create_container(m1, DEFAULT_SUFFIX, 'conflict_parent2') + _create_container(m2, DEFAULT_SUFFIX, 'conflict_parent2') + cont_parent_m1 = _create_container(m1, DEFAULT_SUFFIX, 'conflict_parent3') + cont_parent_m2 = _create_container(m2, DEFAULT_SUFFIX, 'conflict_parent3') + cont_glue_m1 = _create_container(m1, DEFAULT_SUFFIX, 'conflict_parent4') + cont_glue_m2 = _create_container(m2, DEFAULT_SUFFIX, 'conflict_parent4') + + # Create the conflicts + _delete_container(cont_parent_m1) + _create_container(m2, cont_parent_m2.dn, 'conflict_child1') + _delete_container(cont_glue_m1) + _create_container(m2, cont_glue_m2.dn, 'conflict_child2') + + # Resume replication + topo.resume_all_replicas() + time.sleep(5) + + # Test "list" + list_conflicts(m2, None, topo.logcap.log, args) + conflicts = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(conflicts['items']) == 4 + conflict_1_DN = conflicts['items'][0]['dn'] + conflict_2_DN = conflicts['items'][1]['dn'] + conflict_3_DN = conflicts['items'][2]['dn'] + topo.logcap.flush() + + # Test compare + args.DN = conflict_1_DN + cmp_conflict(m2, None, topo.logcap.log, args) + conflicts = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(conflicts['items']) == 2 + topo.logcap.flush() + + # Test delete + del_conflict(m2, None, topo.logcap.log, args) + list_conflicts(m2, None, topo.logcap.log, args) + conflicts = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(conflicts['items']) == 3 + topo.logcap.flush() + + # Test swap + args.DN = conflict_2_DN + swap_conflict(m2, None, topo.logcap.log, args) + list_conflicts(m2, None, topo.logcap.log, args) + conflicts = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(conflicts['items']) == 2 + topo.logcap.flush() + + # Test conflict convert + args.DN = conflict_3_DN + args.new_rdn = "cn=testing convert" + convert_conflict(m2, None, topo.logcap.log, args) + list_conflicts(m2, None, topo.logcap.log, args) + conflicts = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(conflicts['items']) == 1 + topo.logcap.flush() + + # Test list glue entries + list_glue(m2, None, topo.logcap.log, args) + glues = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(glues['items']) == 2 + topo.logcap.flush() + + # Test delete glue entries + args.DN = "cn=conflict_parent3,dc=example,dc=com" + del_glue(m2, None, topo.logcap.log, args) + list_glue(m2, None, topo.logcap.log, args) + glues = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(glues['items']) == 1 + topo.logcap.flush() + + # Test convert glue entries + args.DN = "cn=conflict_parent4,dc=example,dc=com" + convert_glue(m2, None, topo.logcap.log, args) + list_glue(m2, None, topo.logcap.log, args) + glues = json.loads(topo.logcap.outputs[0].getMessage()) + assert len(glues['items']) == 0 + topo.logcap.flush() -- To stop receiving notification emails like this one, please contact the administrator of this repository.