This is an automated email from the git hooks/post-receive script.
firstyear pushed a commit to branch master
in repository 389-ds-base.
commit ae44ed4093ebd02910d23c82a31f13233e2b8c22
Author: William Brown <firstyear(a)redhat.com>
Date: Mon Apr 10 13:47:20 2017 +1000
Ticket 49204 - Fix lower bounds on import autosize + On small VM, autotune breaks the
access of the suffixes
Bug Description:
ldif2db in some cases may set a cache of 0, which may y break imports.
Under memory pressure, the amount of available memory at startup
can be so low that the configured cachememsize will be rejected
(unwilling to perform).
This should leave the cachememsize being "0" (default)
This conduct to be unable to access the suffix pages.
Fix Description:
* autosize set an incorrect percentage which was too high.
* we did not check the lower bound of the allocation
so we now set that we must have a minimum allocation.
* Set entrycache to a minimal value, even if it looks insane
* add a cap on reduction of caches, so we always allocate a few pages
at least, and prevent returning 0 to the caller.
https://pagure.io/389-ds-base/issue/49204
Author: wibrown, tbordaz
Review by: tbordaz (Thanks mate, great work with this :) )
---
ldap/servers/slapd/back-ldbm/dblayer.c | 15 +++++++++-----
ldap/servers/slapd/back-ldbm/ldbm_config.c | 4 ++--
.../servers/slapd/back-ldbm/ldbm_instance_config.c | 23 +++++++++++++++++++---
ldap/servers/slapd/util.c | 20 ++++++++++++++-----
4 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c
b/ldap/servers/slapd/back-ldbm/dblayer.c
index 26a96f6..22e5ffb 100644
--- a/ldap/servers/slapd/back-ldbm/dblayer.c
+++ b/ldap/servers/slapd/back-ldbm/dblayer.c
@@ -1387,12 +1387,17 @@ dblayer_start(struct ldbminfo *li, int dbmode)
/* Sanity check on cache size on platforms which allow us to figure out
* the available phys mem */
slapi_pal_meminfo *mi = spal_meminfo_get();
- if (!util_is_cachesize_sane(mi, &(priv->dblayer_cachesize))) {
+ util_cachesize_result result = util_is_cachesize_sane(mi,
&(priv->dblayer_cachesize));
+ if (result == UTIL_CACHESIZE_ERROR) {
+ slapi_log_err(SLAPI_LOG_CRIT, "dblayer_start", "Unable to
determine if cachesize was valid!!!");
+ } else if (result == UTIL_CACHESIZE_REDUCED) {
+ /* In some cases we saw this go to 0, prevent this. */
+ if (priv->dblayer_cachesize < MINCACHESIZE) {
+ priv->dblayer_cachesize = MINCACHESIZE;
+ }
/* Oops---looks like the admin misconfigured, let's warn them */
- slapi_log_err(SLAPI_LOG_WARNING,"dblayer_start", "Likely
CONFIGURATION ERROR -"
- "dbcachesize is configured to use more than the available "
- "physical memory, decreased to the largest available size
(%"PRIu64" bytes).\n",
- priv->dblayer_cachesize);
+ slapi_log_err(SLAPI_LOG_WARNING, "dblayer_start", "Likely
CONFIGURATION ERROR - dbcachesize is configured to use more than the available "
+ "memory, decreased to (%"PRIu64" bytes).\n",
priv->dblayer_cachesize);
li->li_dbcachesize = priv->dblayer_cachesize;
}
spal_meminfo_destroy(mi);
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.c
b/ldap/servers/slapd/back-ldbm/ldbm_config.c
index 12c79d5..585f9ef 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_config.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_config.c
@@ -1720,9 +1720,9 @@ static config_info ldbm_config[] = {
{CONFIG_DB_DEBUG_CHECKPOINTING, CONFIG_TYPE_ONOFF, "off",
&ldbm_config_db_debug_checkpointing_get, &ldbm_config_db_debug_checkpointing_set,
0},
{CONFIG_DB_HOME_DIRECTORY, CONFIG_TYPE_STRING, "",
&ldbm_config_db_home_directory_get, &ldbm_config_db_home_directory_set, 0},
{CONFIG_IMPORT_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "-1",
&ldbm_config_import_cache_autosize_get, &ldbm_config_import_cache_autosize_set,
CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
- {CONFIG_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "0",
&ldbm_config_cache_autosize_get, &ldbm_config_cache_autosize_set, 0},
+ {CONFIG_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "10",
&ldbm_config_cache_autosize_get, &ldbm_config_cache_autosize_set, 0},
{CONFIG_CACHE_AUTOSIZE_SPLIT, CONFIG_TYPE_INT, "40",
&ldbm_config_cache_autosize_split_get, &ldbm_config_cache_autosize_split_set, 0},
- {CONFIG_IMPORT_CACHESIZE, CONFIG_TYPE_SIZE_T, "20000000",
&ldbm_config_import_cachesize_get, &ldbm_config_import_cachesize_set,
CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
+ {CONFIG_IMPORT_CACHESIZE, CONFIG_TYPE_SIZE_T, "16777216",
&ldbm_config_import_cachesize_get, &ldbm_config_import_cachesize_set,
CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
{CONFIG_IDL_SWITCH, CONFIG_TYPE_STRING, "new",
&ldbm_config_idl_get_idl_new, &ldbm_config_idl_set_tune,
CONFIG_FLAG_ALWAYS_SHOW},
{CONFIG_IDL_UPDATE, CONFIG_TYPE_ONOFF, "on",
&ldbm_config_idl_get_update, &ldbm_config_idl_set_update, 0},
{CONFIG_BYPASS_FILTER_TEST, CONFIG_TYPE_STRING, "on",
&ldbm_config_get_bypass_filter_test, &ldbm_config_set_bypass_filter_test,
CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
b/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
index e2cf544..d7c3c08 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
@@ -101,6 +101,7 @@ ldbm_instance_config_cachememsize_set(void *arg,
int retval = LDAP_SUCCESS;
size_t val = (size_t) value;
uint64_t delta = 0;
+ uint64_t delta_original = 0;
/* Do whatever we can to make sure the data is ok. */
/* There is an error here. We check the new val against our current mem-alloc
@@ -116,18 +117,34 @@ ldbm_instance_config_cachememsize_set(void *arg,
if (apply) {
if (val > inst->inst_cache.c_maxsize) {
delta = val - inst->inst_cache.c_maxsize;
+ delta_original = delta;
util_cachesize_result sane;
slapi_pal_meminfo *mi = spal_meminfo_get();
sane = util_is_cachesize_sane(mi, &delta);
spal_meminfo_destroy(mi);
- if (sane != UTIL_CACHESIZE_VALID){
- slapi_create_errormsg(errorbuf, SLAPI_DSE_RETURNTEXT_SIZE, "Error:
cachememsize value is too large.");
- slapi_log_err(SLAPI_LOG_ERR,
"ldbm_instance_config_cachememsize_set", "cachememsize value is too
large.\n");
+ if (sane == UTIL_CACHESIZE_ERROR){
+ slapi_create_errormsg(errorbuf, SLAPI_DSE_RETURNTEXT_SIZE, "Error:
unable to determine system memory limits.");
+ slapi_log_err(SLAPI_LOG_ERR,
"ldbm_instance_config_cachememsize_set", "Enable to determine system memory
limits.\n");
return LDAP_UNWILLING_TO_PERFORM;
+ } else if (sane == UTIL_CACHESIZE_REDUCED) {
+ slapi_log_err(SLAPI_LOG_WARNING,
"ldbm_instance_config_cachememsize_set", "delta +%"PRIu64" of
request %"PRIu64" reduced to %"PRIu64"\n", delta_original, val,
delta);
+ /*
+ * This works as: value = 100
+ * delta_original to inst, 20;
+ * delta reduced to 5:
+ * 100 - (20 - 5) == 85;
+ * so if you recalculated delta now (val - inst), it would be 5.
+ */
+ val = val - (delta_original - delta);
}
}
+ if (inst->inst_cache.c_maxsize < MINCACHESIZE || val < MINCACHESIZE) {
+ slapi_log_err(SLAPI_LOG_ERR,
"ldbm_instance_config_cachememsize_set", "force a minimal value
%"PRIu64"\n", MINCACHESIZE);
+ /* This value will trigger an autotune next start up, but it should increase
only */
+ val = MINCACHESIZE;
+ }
cache_set_max_size(&(inst->inst_cache), val, CACHE_TYPE_ENTRY);
}
diff --git a/ldap/servers/slapd/util.c b/ldap/servers/slapd/util.c
index 012e83d..4ff6d41 100644
--- a/ldap/servers/slapd/util.c
+++ b/ldap/servers/slapd/util.c
@@ -1468,16 +1468,26 @@ util_is_cachesize_sane(slapi_pal_meminfo *mi, uint64_t
*cachesize)
return UTIL_CACHESIZE_ERROR;
}
+ util_cachesize_result result = UTIL_CACHESIZE_VALID;
slapi_log_err(SLAPI_LOG_TRACE, "util_is_cachesize_sane", "Available
bytes %"PRIu64", requested bytes %"PRIu64"\n",
mi->system_available_bytes, *cachesize);
if (*cachesize > mi->system_available_bytes) {
- /* Since we are ask for more than what's available, we give 3/4 of the
remaining.
+ /* Since we are ask for more than what's available, we give 1/2 of the
remaining.
* the remaining system mem to the cachesize instead, and log a warning
*/
- *cachesize = (mi->system_available_bytes * 0.75);
- slapi_log_err(SLAPI_LOG_TRACE, "util_is_cachesize_sane", "Adjusted
cachesize to %"PRIu64"\n", *cachesize);
- return UTIL_CACHESIZE_REDUCED;
+ uint64_t adjust_cachesize = (mi->system_available_bytes * 0.5);
+ if (adjust_cachesize > *cachesize) {
+ slapi_log_err(SLAPI_LOG_CRIT, "util_is_cachesize_sane",
"Invalid adjusted cachesize is greater than request %"PRIu64,
adjust_cachesize);
+ return UTIL_CACHESIZE_ERROR;
+ }
+ if (adjust_cachesize < (16 * mi->pagesize_bytes)) {
+ /* At minimum respond with 16 pages - that's 64k on x86_64 */
+ adjust_cachesize = 16 * mi->pagesize_bytes;
+ }
+ *cachesize = adjust_cachesize;
+ slapi_log_err(SLAPI_LOG_TRACE, "util_is_cachesize_sane", "Adjusted
cachesize down to %"PRIu64"\n", *cachesize);
+ result = UTIL_CACHESIZE_REDUCED;
}
- return UTIL_CACHESIZE_VALID;
+ return result;
}
long
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.