This is an automated email from the git hooks/post-receive script.
spichugi pushed a commit to branch 389-ds-base-1.4.0
in repository 389-ds-base.
commit e7e6687754c3f7f510503562c090f8dd2676d8cb
Author: Simon Pichugin <spichugi(a)redhat.com>
AuthorDate: Tue Jul 16 21:32:27 2019 +0200
Issue 50497 - Port cl-dump.pl tool to Python using lib389
Bug Description: We're going to deprecate all Perl scripts in 389-ds
so cl-dump.pl should be ported as soon as possible.
Fix Description: Put the tool to dsconf replication dump-changelog.
Preserve all the functionality and output format.
Depricate ChangelogLegacy object.
Move Changelog5 object to replica.py so we can avoid import loops.
Also it makes more sense to have it there because it is part of Replication.
Add ChangelogLDIF object.
Add process_and_dump_changelog() method to Replicas object.
https://pagure.io/389-ds-base/issue/50497
Reviewed by: mreynolds, mhonek, wibrown (Thanks!)
---
.../suites/replication/changelog_trimming_test.py | 3 +-
.../tests/suites/replication/regression_test.py | 10 +-
src/lib389/lib389/changelog.py | 204 -----------------
src/lib389/lib389/cli_conf/replication.py | 41 +++-
src/lib389/lib389/replica.py | 248 ++++++++++++++++++++-
5 files changed, 288 insertions(+), 218 deletions(-)
diff --git a/dirsrvtests/tests/suites/replication/changelog_trimming_test.py
b/dirsrvtests/tests/suites/replication/changelog_trimming_test.py
index e8cc3da..39abc92 100644
--- a/dirsrvtests/tests/suites/replication/changelog_trimming_test.py
+++ b/dirsrvtests/tests/suites/replication/changelog_trimming_test.py
@@ -6,7 +6,7 @@ import time
from lib389._constants import *
from lib389.properties import *
from lib389.topologies import topology_m1 as topo
-from lib389.changelog import Changelog5
+from lib389.replica import Changelog5
from lib389.idm.domain import Domain
DEBUGGING = os.getenv("DEBUGGING", default=False)
@@ -130,4 +130,3 @@ if __name__ == '__main__':
# -s for DEBUG mode
CURRENT_FILE = os.path.realpath(__file__)
pytest.main("-s %s" % CURRENT_FILE)
-
diff --git a/dirsrvtests/tests/suites/replication/regression_test.py
b/dirsrvtests/tests/suites/replication/regression_test.py
index 99280dd..f2bcd3b 100644
--- a/dirsrvtests/tests/suites/replication/regression_test.py
+++ b/dirsrvtests/tests/suites/replication/regression_test.py
@@ -17,8 +17,14 @@ from lib389.agreement import Agreements
from lib389.idm.user import UserAccount
from lib389 import Entry
from lib389.idm.group import Groups, Group
-from lib389.replica import Replicas, ReplicationManager
-from lib389.changelog import Changelog5
+from lib389.idm.domain import Domain
+from lib389.idm.directorymanager import DirectoryManager
+from lib389.replica import Replicas, ReplicationManager, Changelog5
+from lib389.agreement import Agreements
+from lib389 import pid_from_file
+
+
+pytestmark = pytest.mark.tier1
NEW_SUFFIX_NAME = 'test_repl'
NEW_SUFFIX = 'o={}'.format(NEW_SUFFIX_NAME)
diff --git a/src/lib389/lib389/changelog.py b/src/lib389/lib389/changelog.py
deleted file mode 100644
index cb218d4..0000000
--- a/src/lib389/lib389/changelog.py
+++ /dev/null
@@ -1,204 +0,0 @@
-# --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2015 Red Hat, Inc.
-# All rights reserved.
-#
-# License: GPL (version 3 or any later version).
-# See LICENSE for details.
-# --- END COPYRIGHT BLOCK ---
-
-import os
-import ldap
-
-from lib389._constants import *
-from lib389.properties import *
-from lib389 import DirSrv, Entry, InvalidArgumentError
-
-from lib389._mapped_object import DSLdapObject
-from lib389.utils import ds_is_older
-
-
-class Changelog5(DSLdapObject):
- """Represents the Directory Server changelog. This is used for
- replication. Only one changelog is needed for every server.
-
- :param instance: An instance
- :type instance: lib389.DirSrv
- """
-
- def __init__(self, instance, dn='cn=changelog5,cn=config'):
- super(Changelog5, self).__init__(instance, dn)
- self._rdn_attribute = 'cn'
- self._must_attributes = ['cn', 'nsslapd-changelogdir']
- self._create_objectclasses = [
- 'top',
- 'nsChangelogConfig',
- ]
- if ds_is_older('1.4.0'):
- self._create_objectclasses = [
- 'top',
- 'extensibleobject',
- ]
- self._protected = False
-
- def set_max_entries(self, value):
- """Configure the max entries the changelog can hold.
-
- :param value: the number of entries.
- :type value: str
- """
- self.replace('nsslapd-changelogmaxentries', value)
-
- def set_trim_interval(self, value):
- """The time between changelog trims in seconds.
-
- :param value: The time in seconds
- :type value: str
- """
- self.replace('nsslapd-changelogtrim-interval', value)
-
- def set_max_age(self, value):
- """The maximum age of entries in the changelog.
-
- :param value: The age with a time modifier of s, m, h, d, w.
- :type value: str
- """
- self.replace('nsslapd-changelogmaxage', value)
-
-
-class ChangelogLegacy(object):
- """An object that helps to work with changelog entry
-
- :param conn: An instance
- :type conn: lib389.DirSrv
- """
-
- proxied_methods = 'search_s getEntry'.split()
-
- def __init__(self, conn):
- self.conn = conn
- self.log = conn.log
-
- def __getattr__(self, name):
- if name in Changelog.proxied_methods:
- return DirSrv.__getattr__(self.conn, name)
-
- def list(self, suffix=None, changelogdn=DN_CHANGELOG):
- """Get a changelog entry using changelogdn parameter
-
- :param suffix: Not used
- :type suffix: str
- :param changelogdn: DN of the changelog entry, DN_CHANGELOG by default
- :type changelogdn: str
-
- :returns: Search result of the replica agreements.
- Enpty list if nothing was found
- """
-
- base = changelogdn
- filtr = "(objectclass=extensibleobject)"
-
- # now do the effective search
- try:
- ents = self.conn.search_s(base, ldap.SCOPE_BASE, filtr)
- except ldap.NO_SUCH_OBJECT:
- # There are no objects to select from, se we return an empty array
- # as we do in DSLdapObjects
- ents = []
- return ents
-
- def create(self, dbname=DEFAULT_CHANGELOG_DB):
- """Add and return the replication changelog entry.
-
- :param dbname: Database name, it will be used for creating
- a changelog dir path
- :type dbname: str
- """
-
- dn = DN_CHANGELOG
- attribute, changelog_name = dn.split(",")[0].split("=", 1)
- dirpath = os.path.join(os.path.dirname(self.conn.dbdir), dbname)
- entry = Entry(dn)
- entry.update({
- 'objectclass': ("top", "extensibleobject"),
- CHANGELOG_PROPNAME_TO_ATTRNAME[CHANGELOG_NAME]: changelog_name,
- CHANGELOG_PROPNAME_TO_ATTRNAME[CHANGELOG_DIR]: dirpath
- })
- self.log.debug("adding changelog entry: %r", entry)
- self.conn.changelogdir = dirpath
- try:
- self.conn.add_s(entry)
- except ldap.ALREADY_EXISTS:
- self.log.warning("entry %s already exists", dn)
- return dn
-
- def delete(self):
- """Delete the changelog entry
-
- :raises: LDAPError - failed to delete changelog entry
- """
-
- try:
- self.conn.delete_s(DN_CHANGELOG)
- except ldap.LDAPError as e:
- self.log.error('Failed to delete the changelog: %s', e)
- raise
-
- def setProperties(self, changelogdn=None, properties=None):
- """Set the properties of the changelog entry.
-
- :param changelogdn: DN of the changelog
- :type changelogdn: str
- :param properties: Dictionary of properties
- :type properties: dict
-
- :returns: None
- :raises: - ValueError - if invalid properties
- - ValueError - if changelog entry is not found
- - InvalidArgumentError - changelog DN is missing
-
- :supported properties are:
- CHANGELOG_NAME, CHANGELOG_DIR, CHANGELOG_MAXAGE,
- CHANGELOG_MAXENTRIES, CHANGELOG_TRIM_INTERVAL,
- CHANGELOG_COMPACT_INTV, CHANGELOG_CONCURRENT_WRITES,
- CHANGELOG_ENCRYPT_ALG, CHANGELOG_SYM_KEY
- """
-
- if not changelogdn:
- raise InvalidArgumentError("changelog DN is missing")
-
- ents = self.conn.changelog.list(changelogdn=changelogdn)
- if len(ents) != 1:
- raise ValueError("Changelog entry not found: %s" % changelogdn)
-
- # check that the given properties are valid
- for prop in properties:
- # skip the prefix to add/del value
- if not inProperties(prop, CHANGELOG_PROPNAME_TO_ATTRNAME):
- raise ValueError("unknown property: %s" % prop)
-
- # build the MODS
- mods = []
- for prop in properties:
- # take the operation type from the property name
- val = rawProperty(prop)
- if str(prop).startswith('+'):
- op = ldap.MOD_ADD
- elif str(prop).startswith('-'):
- op = ldap.MOD_DELETE
- else:
- op = ldap.MOD_REPLACE
-
- mods.append((op, CHANGELOG_PROPNAME_TO_ATTRNAME[val],
- properties[prop]))
-
- # that is fine now to apply the MOD
- self.conn.modify_s(ents[0].dn, mods)
-
- def getProperties(self, changelogdn=None, properties=None):
- """Get a dictionary of the requested properties.
- If properties parameter is missing, it returns all the properties.
-
- NotImplemented
- """
-
- raise NotImplemented
diff --git a/src/lib389/lib389/cli_conf/replication.py
b/src/lib389/lib389/cli_conf/replication.py
index b25eba6..3e147e9 100644
--- a/src/lib389/lib389/cli_conf/replication.py
+++ b/src/lib389/lib389/cli_conf/replication.py
@@ -6,13 +6,16 @@
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---
+import logging
+import time
+import base64
+import os
import json
import ldap
from getpass import getpass
from lib389._constants import *
-from lib389.changelog import Changelog5
-from lib389.utils import is_a_dn
-from lib389.replica import Replicas, BootstrapReplicationManager
+from lib389.utils import is_a_dn, ensure_str
+from lib389.replica import Replicas, BootstrapReplicationManager, RUV, Changelog5,
ChangelogLDIF
from lib389.tasks import CleanAllRUVTask, AbortCleanAllRUVTask
from lib389._mapped_object import DSLdapObjects
@@ -885,6 +888,25 @@ def list_abort_cleanallruv(inst, basedn, log, args):
log.info("No CleanAllRUV abort tasks found")
+def dump_cl(inst, basedn, log, args):
+ if args.output_file:
+ fh = logging.FileHandler(args.output_file, mode='w')
+ log.addHandler(fh)
+ replicas = Replicas(inst)
+ if not args.changelog_ldif:
+ replicas.process_and_dump_changelog(replica_roots=args.replica_roots,
csn_only=args.csn_only)
+ else:
+ try:
+ assert os.path.exists(args.changelog_ldif)
+ except AssertionError:
+ raise FileNotFoundError(f"File {args.changelog_ldif} was not
found")
+ cl_ldif = ChangelogLDIF(args.changelog_ldif, log)
+ if args.csn_only:
+ cl_ldif.grep_csn()
+ else:
+ cl_ldif.decode()
+
+
def create_parser(subparsers):
############################################
@@ -971,6 +993,18 @@ def create_parser(subparsers):
repl_get_cl = repl_subcommands.add_parser('get-changelog', help='Display
replication changelog attributes.')
repl_get_cl.set_defaults(func=get_cl)
+ repl_set_cl = repl_subcommands.add_parser('dump-changelog', help='Decode
Directory Server replication change log and dump it to an LDIF')
+ repl_set_cl.set_defaults(func=dump_cl)
+ repl_set_cl.add_argument('-c', '--csn-only',
action='store_true',
+ help="Dump and interpret CSN only. This option can be
used with or without -i option.")
+ repl_set_cl.add_argument('-i', '--changelog-ldif',
+ help="If you already have a ldif-like changelog, but
the changes in that file are encoded,"
+ " you may use this option to decode that ldif-like
changelog. It should be base64 encoded.")
+ repl_set_cl.add_argument('-o', '--output-file', help="Path name
for the final result. Default to STDOUT if omitted.")
+ repl_set_cl.add_argument('-r', '--replica-roots',
nargs="+",
+ help="Specify replica roots whose changelog you want to
dump. The replica "
+ "roots may be seperated by comma. All the replica
roots would be dumped if the option is omitted.")
+
repl_set_parser = repl_subcommands.add_parser('set', help='Set an
attribute in the replication configuration')
repl_set_parser.set_defaults(func=set_repl_config)
repl_set_parser.add_argument('--suffix', required=True, help='The DN of
the replication suffix')
@@ -1264,4 +1298,3 @@ def create_parser(subparsers):
task_abort_cleanallruv_list =
task_subcommands.add_parser('list-abortruv-tasks', help='List all the running
CleanAllRUV abort Tasks')
task_abort_cleanallruv_list.set_defaults(func=list_abort_cleanallruv)
task_abort_cleanallruv_list.add_argument('--suffix', help="List only
tasks from for suffix")
-
diff --git a/src/lib389/lib389/replica.py b/src/lib389/lib389/replica.py
index 456e493..a67a829 100644
--- a/src/lib389/lib389/replica.py
+++ b/src/lib389/lib389/replica.py
@@ -6,9 +6,12 @@
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---
+import os
+import base64
import ldap
import decimal
import time
+import datetime
import logging
import uuid
import json
@@ -22,7 +25,6 @@ from lib389._mapped_object import DSLdapObjects, DSLdapObject
from lib389.passwd import password_generate
from lib389.mappingTree import MappingTrees
from lib389.agreement import Agreements
-from lib389.changelog import Changelog5
from lib389.tombstone import Tombstones
from lib389.idm.domain import Domain
@@ -815,15 +817,19 @@ class RUV(object):
:type logger: logging object
"""
- def __init__(self, ruvs, logger=None):
+ def __init__(self, ruvs=[], logger=None):
if logger is not None:
self._log = logger
else:
self._log = logging.getLogger(__name__)
self._rids = []
- self._rid_csn = {}
self._rid_url = {}
+ self._rid_rawruv = {}
+ self._rid_csn = {}
+ self._rid_maxcsn = {}
+ self._rid_modts = {}
self._data_generation = None
+ self._data_generation_csn = None
# Process the array of data
for r in ruvs:
pr = r.replace('{', '').replace('}',
'').split(' ')
@@ -836,10 +842,66 @@ class RUV(object):
rid = pr[1]
self._rids.append(rid)
self._rid_url[rid] = pr[2]
+ self._rid_rawruv[rid] = r
try:
- self._rid_csn[rid] = pr[4]
+ self._rid_csn[rid] = pr[3]
except IndexError:
self._rid_csn[rid] = '00000000000000000000'
+ try:
+ self._rid_maxcsn[rid] = pr[4]
+ except IndexError:
+ self._rid_maxcsn[rid] = '00000000000000000000'
+ try:
+ self._rid_modts[rid] = pr[5]
+ except IndexError:
+ self._rid_modts[rid] = '00000000'
+
+ @staticmethod
+ def parse_csn(csn):
+ """Parse CSN into human readable format '1970-01-31
00:00:00'
+
+ :param csn: the CSN to format
+ :type csn: str
+ :returns: str
+ """
+ if len(csn) != 20 or len(csn) != 8 and not isinstance(csn, str):
+ ValueError("Wrong CSN value was supplied")
+
+ timestamp = int(csn[:8], 16)
+ time_str = datetime.datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d
%H:%M:%S')
+ # We are parsing shorter CSN which contains only timestamp
+ if len(csn) == 8:
+ return time_str
+ else:
+ seq = int(csn[8:12], 16)
+ subseq = int(csn[16:20], 16)
+ if seq != 0 or subseq != 0:
+ return f"{time_str} {str(seq)} {str(subseq)}"
+ else:
+ return f"{time_str}"
+
+ def format_ruv(self):
+ """Parse RUV into human readable format
+
+ :returns: dict
+ """
+ result = {}
+ if self._data_generation:
+ result["data_generation"] = {"name":
self._data_generation,
+ "value": self._data_generation_csn}
+ else:
+ result["data_generation"] = None
+
+ ruvs = []
+ for rid in self._rids:
+ ruvs.append({"raw_ruv": self._rid_rawruv.get(rid),
+ "rid": rid,
+ "url": self._rid_url.get(rid),
+ "csn": parse_csn(self._rid_csn.get(rid,
'00000000000000000000')),
+ "maxcsn": parse_csn(self._rid_maxcsn.get(rid,
'00000000000000000000')),
+ "modts": parse_csn(self._rid_modts.get(rid,
'00000000'))})
+ result["ruvs"] = ruvs
+ return result
def alloc_rid(self):
"""Based on the RUV, determine an available RID for the
replication
@@ -880,6 +942,133 @@ class RUV(object):
return True
+class ChangelogLDIF(object):
+ def __init__(self, file_path, logger=None):
+ """A class for working with Changelog LDIF file
+
+ :param file_path: LDIF file path
+ :type file_path: str
+ :param logger: A logging object
+ :type logger: logging.Logger
+ """
+
+ if logger is not None:
+ self._log = logger
+ else:
+ self._log = logging.getLogger(__name__)
+ self.file_path = file_path
+
+ def grep_csn(self):
+ """Grep and interpret CSNs
+
+ :param file: LDIF file path
+ :type file: str
+ """
+
+ self._log.info(f"# LDIF File: {self.file_path}")
+ with open(self.file_path) as f:
+ for line in f.readlines():
+ if "ruv:" in line or "csn:" in line:
+ csn = ""
+ maxcsn = ""
+ modts = ""
+ line = line.split("\n")[0]
+ if "ruv:" in line:
+ ruv = RUV([line.split("ruv: ")[1]])
+ ruv_dict = ruv.parse_ruv()
+ csn = ruv_dict["csn"]
+ maxcsn = ruv_dict["maxcsn"]
+ modts = ruv_dict["modts"]
+ elif "csn:" in line:
+ csn = RUV().parse_csn(line.split("csn: ")[1])
+ if maxcsn or modts:
+ self._log.info(f'{line} ({csn}')
+ if maxcsn:
+ self._log.info(f"; {maxcsn}")
+ if modts:
+ self._log.info(f"; {modts}")
+ self._log.info(")")
+ else:
+ self._log.info(f"{line} ({csn})")
+
+ def decode(self):
+ """Decode the changelog
+
+ :param file: LDIF file path
+ :type file: str
+ """
+
+ self._log.info(f"# LDIF File: {self.file_path}")
+ with open(self.file_path) as f:
+ encoded_str = ""
+ for line in f.readlines():
+ if line.startswith("change::") or
line.startswith("changes::"):
+ self._log.info("change::")
+ try:
+ encoded_str = line.split("change:: ")[1]
+ except IndexError:
+ encoded_str = line.split("changes:: ")[1]
+ continue
+ if not encoded_str:
+ self._log.info(line.split('\n')[0])
+ continue
+ if line == "\n":
+ decoded_str = ensure_str(base64.b64decode(encoded_str))
+ self._log.info(decoded_str)
+ encoded_str = ""
+ continue
+ encoded_str += line
+
+
+class Changelog5(DSLdapObject):
+ """Represents the Directory Server changelog. This is used for
+ replication. Only one changelog is needed for every server.
+
+ :param instance: An instance
+ :type instance: lib389.DirSrv
+ """
+
+ def __init__(self, instance, dn='cn=changelog5,cn=config'):
+ super(Changelog5, self).__init__(instance, dn)
+ self._rdn_attribute = 'cn'
+ self._must_attributes = ['cn', 'nsslapd-changelogdir']
+ self._create_objectclasses = [
+ 'top',
+ 'nsChangelogConfig',
+ ]
+ if ds_is_older('1.4.0'):
+ self._create_objectclasses = [
+ 'top',
+ 'extensibleobject',
+ ]
+ self._protected = False
+
+ def set_max_entries(self, value):
+ """Configure the max entries the changelog can hold.
+
+ :param value: the number of entries.
+ :type value: str
+ """
+ self.replace('nsslapd-changelogmaxentries', value)
+
+ def set_trim_interval(self, value):
+ """The time between changelog trims in seconds.
+
+ :param value: The time in seconds
+ :type value: str
+ """
+ self.replace('nsslapd-changelogtrim-interval', value)
+
+ def set_max_age(self, value):
+ """The maximum age of entries in the changelog.
+
+ :param value: The age with a time modifier of s, m, h, d, w.
+ :type value: str
+ """
+
+ self.replace('nsslapd-changelogmaxage', value)
+
+
class Replica(DSLdapObject):
"""Replica DSLdapObject with:
- must attributes = ['cn', 'nsDS5ReplicaType',
'nsDS5ReplicaRoot',
@@ -1307,6 +1496,55 @@ class Replicas(DSLdapObjects):
replica._populate_suffix()
return replica
+ def process_and_dump_changelog(self, replica_roots=[], csn_only=False):
+ """Dump and decode Directory Server replication change log
+
+ :param replica_roots: Replica suffixes that need to be processed
+ :type replica_roots: list of str
+ :param csn_only: Grep only the CSNs from the file
+ :type csn_only: bool
+ """
+
+ repl_roots = []
+ try:
+ cl = Changelog5(self._instance)
+ cl_dir = cl.get_attr_val_utf8_l("nsslapd-changelogdir")
+ except ldap.NO_SUCH_OBJECT:
+ raise ValueError("Changelog entry was not found. Probably, the
replication is not enabled on this instance")
+
+ # Get all the replicas on the server if --replica-roots option is not specified
+ if not replica_roots:
+ for replica in self.list():
+
repl_roots.append(replica.get_attr_val_utf8("nsDS5ReplicaRoot"))
+ else:
+ for repl_root in replica_roots:
+ repl_roots.append(repl_root)
+
+ # Dump the changelog for the replica
+ for repl_root in repl_roots:
+ got_ldif = False
+ current_time = time.time()
+ replica = self.get(repl_root)
+ self._log.info(f"# Replica Root: {repl_root}")
+ replica.replace("nsDS5Task", 'CL2LDIF')
+
+ # Decode the dumped changelog
+ for file in [i for i in os.listdir(cl_dir) if i.endswith('.ldif')]:
+ file_path = os.path.join(cl_dir, file)
+ # Skip older ldif files
+ if os.path.getmtime(file_path) < current_time:
+ continue
+ got_ldif = True
+ cl_ldif = ChangelogLDIF(file_path, self._log)
+
+ if csn_only:
+ cl_ldif.grep_csn()
+ else:
+ cl_ldif.decode()
+ os.rename(file_path, f'{file_path}.done')
+ if not got_ldif:
+ self._log.info("LDIF file: Not found")
+
class BootstrapReplicationManager(DSLdapObject):
"""A Replication Manager credential for bootstrapping the repl
process.
@@ -1989,5 +2227,3 @@ class ReplicationManager(object):
replicas = Replicas(instance)
replica = replicas.get(self._suffix)
return replica.get_rid()
-
-
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.