diff --git a/include/zbxalgo.h b/include/zbxalgo.h index 1779138d8c..851d029918 100644 --- a/include/zbxalgo.h +++ b/include/zbxalgo.h @@ -184,6 +184,8 @@ void zbx_hashset_iter_reset(zbx_hashset_t *hs, zbx_hashset_iter_t *iter); void *zbx_hashset_iter_next(zbx_hashset_iter_t *iter); void zbx_hashset_iter_remove(zbx_hashset_iter_t *iter); +void zbx_hashset_copy(zbx_hashset_t *dst, zbx_hashset_t *src, size_t size); + /* hashmap */ /* currently, we only have a very specialized hashmap */ diff --git a/src/libs/zbxalgo/hashset.c b/src/libs/zbxalgo/hashset.c index 01a0deab89..714cbb5912 100644 --- a/src/libs/zbxalgo/hashset.c +++ b/src/libs/zbxalgo/hashset.c @@ -444,3 +444,30 @@ void zbx_hashset_iter_remove(zbx_hashset_iter_t *iter) iter->entry = prev_entry; } } + +void zbx_hashset_copy(zbx_hashset_t *dst, zbx_hashset_t *src, size_t size) +{ + int i; + ZBX_HASHSET_ENTRY_T *entry, **ref; + + *dst = *src; + + dst->slots = (ZBX_HASHSET_ENTRY_T **)dst->mem_malloc_func(NULL, dst->num_slots * sizeof(ZBX_HASHSET_ENTRY_T *)); + memset(dst->slots, 0, dst->num_slots * sizeof(ZBX_HASHSET_ENTRY_T *)); + + for (i = 0; i < src->num_slots; i++) + { + if (0 == src->slots[i]) + continue; + + for (ref = &dst->slots[i], entry = src->slots[i]; NULL != entry; entry = entry->next) + { + *ref = (ZBX_HASHSET_ENTRY_T *)src->mem_malloc_func(NULL, ZBX_HASHSET_ENTRY_OFFSET + size); + memcpy(*ref, entry, ZBX_HASHSET_ENTRY_OFFSET + size); + ref = &(*ref)->next; + } + } +} + + + diff --git a/src/libs/zbxdbcache/Makefile.am b/src/libs/zbxdbcache/Makefile.am index d3345a8ca7..2ab045bf49 100644 --- a/src/libs/zbxdbcache/Makefile.am +++ b/src/libs/zbxdbcache/Makefile.am @@ -11,7 +11,8 @@ libzbxdbcache_a_SOURCES = \ dbsync.c \ dbsync.h \ valuecache.c \ - valuecache.h + valuecache.h \ + user_macro.c libzbxdbcache_a_CFLAGS = \ -I$(top_srcdir)/src/zabbix_server/ \ diff --git a/src/libs/zbxdbcache/dbconfig.c b/src/libs/zbxdbcache/dbconfig.c index 608c6e2fac..cbd251a567 100644 --- a/src/libs/zbxdbcache/dbconfig.c +++ b/src/libs/zbxdbcache/dbconfig.c @@ -81,7 +81,7 @@ typedef int (*zbx_value_validator_func_t)(const char *macro, const char *value, ZBX_DC_CONFIG *config = NULL; zbx_rwlock_t config_lock = ZBX_RWLOCK_NULL; -static zbx_mem_info_t *config_mem; +zbx_mem_info_t *config_mem; extern unsigned char program_type; extern int CONFIG_TIMER_FORKS; @@ -823,7 +823,6 @@ static zbx_dc_kv_t *config_kvs_path_add(const char *path, const char *key) if (NULL == kv) { - DCstrpool_replace(0, &kv_local.key, key); kv_local.value = NULL; kv_local.refcount = 0; diff --git a/src/libs/zbxdbcache/user_macro.c b/src/libs/zbxdbcache/user_macro.c new file mode 100644 index 0000000000..26988e212c --- /dev/null +++ b/src/libs/zbxdbcache/user_macro.c @@ -0,0 +1,1139 @@ +/* +** Zabbix +** Copyright (C) 2001-2022 Zabbix SIA +** +** This program is free software; you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation; either version 2 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software +** Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +**/ + +#include "common.h" +#include "zbxalgo.h" +#include "memalloc.h" +#include "zbxalgo.h" +#include "../zbxalgo/vectorimpl.h" +#include "dbsync.h" + +//extern zbx_mem_info_t *config_mem; + +//ZBX_MEM_FUNC_IMPL(__config, config_mem) + +static void *__config_mem_malloc_func(void *p, size_t s) +{ + return zbx_malloc(p, s); +} + +static void *__config_mem_realloc_func(void *p, size_t s) +{ + return zbx_realloc(p, s); +} + +static void __config_mem_free_func(void *p) +{ + zbx_free(p); +} + + +// WDN strpool heap implementation + +#define zbx_strpool_intern(s) um_strpool_intern(s) +#define zbx_strpool_acquire(s) um_strpool_acquire(s) +#define zbx_strpool_release(s) um_strpool_release(s) + +static zbx_hashset_t strpool; + +#define REFCOUNT_FIELD_SIZE sizeof(zbx_uint32_t) + +static zbx_hash_t __config_strpool_hash(const void *data) +{ + return ZBX_DEFAULT_STRING_HASH_FUNC((char *)data + REFCOUNT_FIELD_SIZE); +} + +static int __config_strpool_compare(const void *d1, const void *d2) +{ + return strcmp((char *)d1 + REFCOUNT_FIELD_SIZE, (char *)d2 + REFCOUNT_FIELD_SIZE); +} + +static void um_strpool_init() +{ + zbx_hashset_create(&strpool, 100, __config_strpool_hash, __config_strpool_compare); +} + +static void um_strpool_destroy() +{ + zbx_hashset_destroy(&strpool); +} + +static const char *um_strpool_intern(const char *str) +{ + void *record; + zbx_uint32_t *refcount; + + record = zbx_hashset_search(&strpool, str - REFCOUNT_FIELD_SIZE); + + if (NULL == record) + { + record = zbx_hashset_insert_ext(&strpool, str - REFCOUNT_FIELD_SIZE, + REFCOUNT_FIELD_SIZE + strlen(str) + 1, REFCOUNT_FIELD_SIZE); + *(zbx_uint32_t *)record = 0; + } + + refcount = (zbx_uint32_t *)record; + (*refcount)++; + + return (char *)record + REFCOUNT_FIELD_SIZE; +} + +static void um_strpool_release(const char *str) +{ + zbx_uint32_t *refcount; + + refcount = (zbx_uint32_t *)(str - REFCOUNT_FIELD_SIZE); + if (0 == --(*refcount)) + zbx_hashset_remove(&strpool, str - REFCOUNT_FIELD_SIZE); +} + +static const char *um_strpool_acquire(const char *str) +{ + zbx_uint32_t *refcount; + + refcount = (zbx_uint32_t *)(str - REFCOUNT_FIELD_SIZE); + (*refcount)++; + + return str; +} + +typedef struct +{ + zbx_uint64_t macroid; + zbx_uint64_t hostid; + const char *name; + const char *context; + const char *value; + zbx_uint32_t refcount; + unsigned char type; + unsigned char context_op; +} +zbx_um_t; + +ZBX_PTR_VECTOR_DECL(um, zbx_um_t *) +ZBX_PTR_VECTOR_IMPL(um, zbx_um_t *) + +typedef struct +{ + zbx_uint64_t hostid; + zbx_vector_uint64_t templateids; + zbx_uint32_t refcount; +} +zbx_um_host_t; + +typedef struct +{ + zbx_uint64_t hostid; + const char *macro; + zbx_vector_um_t values; + zbx_uint32_t refcount; +} +zbx_um_index_t; + +typedef struct +{ + zbx_hashset_t macros; + zbx_hashset_t hosts; + zbx_hashset_t index; + zbx_uint32_t refcount; +} +zbx_um_cache_t; + + +static zbx_um_cache_t *um_cache_dup(zbx_um_cache_t *cache) +{ + zbx_um_cache_t *dup; + zbx_um_t **macro; + zbx_um_host_t **host; + zbx_um_index_t **index; + zbx_hashset_iter_t iter; + + dup = (zbx_um_cache_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_cache_t)); + dup->refcount = 1; + + zbx_hashset_copy(&dup->macros, &cache->macros, sizeof(zbx_um_t *)); + zbx_hashset_iter_reset(&dup->macros, &iter); + while (NULL != (macro = (zbx_um_t **)zbx_hashset_iter_next(&iter))) + (*macro)->refcount++; + + zbx_hashset_copy(&dup->hosts, &cache->hosts, sizeof(zbx_um_host_t *)); + zbx_hashset_iter_reset(&dup->hosts, &iter); + while (NULL != (host = (zbx_um_host_t **)zbx_hashset_iter_next(&iter))) + (*host)->refcount++; + + zbx_hashset_copy(&dup->index, &cache->index, sizeof(zbx_um_index_t *)); + zbx_hashset_iter_reset(&dup->index, &iter); + while (NULL != (index = (zbx_um_index_t **)zbx_hashset_iter_next(&iter))) + (*index)->refcount++; + + return dup; +} + +static zbx_hash_t um_macro_hash_func(const void *d) +{ + const zbx_um_t *macro = *(const zbx_um_t * const *)d; + + return ZBX_DEFAULT_UINT64_HASH_FUNC(¯o->macroid); +} + +static int um_macro_compare_func(const void *d1, const void *d2) +{ + const zbx_um_t *m1 = *(const zbx_um_t * const *)d1; + const zbx_um_t *m2 = *(const zbx_um_t * const *)d2; + + ZBX_RETURN_IF_NOT_EQUAL(m1->macroid, m2->macroid); + return 0; +} + +static zbx_hash_t um_index_hash_func(const void *d) +{ + const zbx_um_index_t *index = *(const zbx_um_index_t * const *)d; + zbx_hash_t hash; + + hash = ZBX_DEFAULT_UINT64_HASH_FUNC(&index->hostid); + return ZBX_DEFAULT_STRING_HASH_ALGO(index->macro, strlen(index->macro), hash); +} + +static int um_index_compare_func(const void *d1, const void *d2) +{ + const zbx_um_index_t *i1 = *(const zbx_um_index_t * const *)d1; + const zbx_um_index_t *i2 = *(const zbx_um_index_t * const *)d2; + + ZBX_RETURN_IF_NOT_EQUAL(i1->hostid, i2->hostid); + return strcmp(i1->macro, i2->macro); +} + +static zbx_hash_t um_host_hash_func(const void *d) +{ + const zbx_um_host_t *host = *(const zbx_um_host_t * const *)d; + + return ZBX_DEFAULT_UINT64_HASH_FUNC(&host->hostid); +} + +static int um_host_compare_func(const void *d1, const void *d2) +{ + const zbx_um_host_t *h1 = *(const zbx_um_host_t * const *)d1; + const zbx_um_host_t *h2 = *(const zbx_um_host_t * const *)d2; + + ZBX_RETURN_IF_NOT_EQUAL(h1->hostid, h2->hostid); + return 0; +} + +static zbx_um_cache_t *um_cache_create() +{ + zbx_um_cache_t *cache; + + cache = (zbx_um_cache_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_cache_t)); + + cache->refcount = 1; + + zbx_hashset_create(&cache->macros, 100, um_macro_hash_func, um_macro_compare_func); + zbx_hashset_create(&cache->index, 100, um_index_hash_func, um_index_compare_func); + zbx_hashset_create(&cache->hosts, 10, um_host_hash_func, um_host_compare_func); + + return cache; +} + +static void um_macro_release(zbx_um_t *macro) +{ + if (0 != --macro->refcount) + return; + + zbx_strpool_release(macro->name); + if (NULL != macro->context) + zbx_strpool_release(macro->context); + zbx_strpool_release(macro->value); + + __config_mem_free_func(macro); +} + +static void um_host_release(zbx_um_host_t *host) +{ + if (0 != --host->refcount) + return; + + zbx_vector_uint64_destroy(&host->templateids); + __config_mem_free_func(host); +} + +static void um_index_release(zbx_um_index_t *index) +{ + int i; + + if (0 != --index->refcount) + return; + + zbx_strpool_release(index->macro); + + for (i = 0; i < index->values.values_num; i++) + um_macro_release(index->values.values[i]); + zbx_vector_um_destroy(&index->values); + + __config_mem_free_func(index); +} + +static void um_cache_release(zbx_um_cache_t *cache) +{ + zbx_um_t **macro; + zbx_um_host_t **host; + zbx_um_index_t **index; + zbx_hashset_iter_t iter; + + if (0 != --cache->refcount) + return; + + zbx_hashset_iter_reset(&cache->macros, &iter); + while (NULL != (macro = (zbx_um_t **)zbx_hashset_iter_next(&iter))) + um_macro_release(*macro); + zbx_hashset_destroy(&cache->macros); + + zbx_hashset_iter_reset(&cache->hosts, &iter); + while (NULL != (host = (zbx_um_host_t **)zbx_hashset_iter_next(&iter))) + um_host_release(*host); + zbx_hashset_destroy(&cache->hosts); + + zbx_hashset_iter_reset(&cache->index, &iter); + while (NULL != (index = (zbx_um_index_t **)zbx_hashset_iter_next(&iter))) + um_index_release(*index); + zbx_hashset_destroy(&cache->index); + + __config_mem_free_func(cache); +} + +static zbx_um_t *um_macro_dup(zbx_um_t *macro) +{ + zbx_um_t *dup; + + dup = (zbx_um_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_t)); + dup->macroid = macro->macroid; + dup->hostid = macro->hostid; + dup->name = zbx_strpool_acquire(macro->name); + dup->context = (NULL != macro->context ? zbx_strpool_acquire(macro->context) : NULL); + dup->value = zbx_strpool_acquire(macro->value); + dup->type = macro->type; + dup->context_op = macro->context_op; + dup->refcount = 1; + + return dup; +} + +static zbx_um_index_t *um_index_dup(zbx_um_index_t *index) +{ + zbx_um_index_t *dup; + int i; + + dup = (zbx_um_index_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_index_t)); + dup->hostid = index->hostid; + dup->macro = um_strpool_acquire(index->macro); + dup->refcount = 1; + zbx_vector_um_create_ext(&dup->values, __config_mem_malloc_func, __config_mem_realloc_func, + __config_mem_free_func); + zbx_vector_um_append_array(&dup->values, index->values.values, index->values.values_num); + + for (i = 0; i < dup->values.values_num; i++) + dup->values.values[i]->refcount++; + + return dup; +} + +static zbx_um_host_t *um_host_dup(zbx_um_host_t *host) +{ + zbx_um_host_t *dup; + int i; + + dup = (zbx_um_host_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_host_t)); + dup->hostid = host->hostid; + dup->refcount = 1; + zbx_vector_uint64_create_ext(&dup->templateids, __config_mem_malloc_func, __config_mem_realloc_func, + __config_mem_free_func); + zbx_vector_uint64_append_array(&dup->templateids, host->templateids.values, host->templateids.values_num); + + return dup; +} + +static void um_index_remove_macro(zbx_um_index_t *index, zbx_um_t *macro) +{ + int i; + + for (i = 0; i < index->values.values_num; i++) + { + if (index->values.values[i] == macro) + { + zbx_vector_um_remove_noorder(&index->values, i); + um_macro_release(macro); + break; + } + } +} + +static zbx_um_index_t *um_macro_create_index(zbx_um_t *macro) +{ + zbx_um_index_t *index; + + index = (zbx_um_index_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_index_t)); + index->hostid = macro->hostid; + index->macro = zbx_strpool_acquire(macro->name); + index->refcount = 1; + zbx_vector_um_create(&index->values); + + return index; +} + +static int um_macro_is_locked(const zbx_um_t *macro) +{ + /* macro is referred by cache and index, so refcount 2 means nothing else is using it */ + return 2 != macro->refcount ? SUCCEED : FAIL; +} + +static int um_index_is_locked(const zbx_um_index_t *index) +{ + return 1 != index->refcount ? SUCCEED : FAIL; +} + +static int um_host_is_locked(const zbx_um_host_t *host) +{ + return 1 != host->refcount ? SUCCEED : FAIL; +} + +static void um_index_acquire(zbx_um_index_t **pindex) +{ + if (SUCCEED == um_index_is_locked(*pindex)) + { + zbx_um_index_t *index; + + index = um_index_dup(*pindex); + um_index_release(*pindex); + *pindex = index; + } +} + +static void um_cache_deindex_macro(zbx_um_cache_t *cache, zbx_um_t *macro) +{ + zbx_um_index_t **pindex, index_local, *pindex_local = &index_local; + int i; + + index_local.hostid = macro->hostid; + index_local.macro = macro->name; + + if (NULL == (pindex = (zbx_um_index_t **)zbx_hashset_search(&cache->index, &pindex_local))) + return; + + um_index_acquire(pindex); + + for (i = 0; i < (*pindex)->values.values_num; i++) + { + if (macro == (*pindex)->values.values[i]) + { + um_macro_release((*pindex)->values.values[i]); + zbx_vector_um_remove_noorder(&(*pindex)->values, i); + if (0 == (*pindex)->values.values_num) + { + um_index_release(*pindex); + zbx_hashset_remove_direct(&cache->index, pindex); + } + break; + } + } + +} + +static void um_cache_index_macro(zbx_um_cache_t *cache, zbx_um_t *macro) +{ + zbx_um_index_t **pindex, *index, index_local, *pindex_local = &index_local; + + index_local.hostid = macro->hostid; + index_local.macro = macro->name; + + if (NULL == (pindex = (zbx_um_index_t **)zbx_hashset_search(&cache->index, &pindex_local))) + { + index = (zbx_um_index_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_index_t)); + index->hostid = macro->hostid; + index->macro = zbx_strpool_acquire(macro->name); + index->refcount = 1; + zbx_vector_um_create_ext(&index->values, __config_mem_malloc_func, __config_mem_realloc_func, + __config_mem_free_func); + + pindex = zbx_hashset_insert(&cache->index, &index, sizeof(index)); + } + else + um_index_acquire(pindex); + + zbx_vector_um_append(&(*pindex)->values, macro); + macro->refcount++; +} + +#define ZBX_UM_MACRO_UPDATE_HOSTID 0x0001 +#define ZBX_UM_MACRO_UPDATE_NAME 0x0002 +#define ZBX_UM_MACRO_UPDATE_CONTEXT 0x0004 +#define ZBX_UM_MACRO_UPDATE_VALUE 0x0008 + +#define ZBX_UM_INDEX_UPDATE (ZBX_UM_MACRO_UPDATE_HOSTID | ZBX_UM_MACRO_UPDATE_NAME) + +static void um_cache_sync_macros(zbx_um_cache_t *cache, zbx_dbsync_t *sync, int offset) +{ + unsigned char tag; + int ret; + zbx_uint64_t rowid, hostid = 0, *pmacroid = &rowid; + char **row; + zbx_um_t **pmacro; + + while (SUCCEED == (ret = zbx_dbsync_next(sync, &rowid, &row, &tag))) + { + char *name = NULL, *context = NULL; + const char *dc_name, *dc_context, *dc_value; + unsigned char context_op; + zbx_um_t *macro; + int index_macro = 1; + + /* removed rows will be always at the end of sync list */ + if (ZBX_DBSYNC_ROW_REMOVE == tag) + break; + + if (SUCCEED != zbx_user_macro_parse_dyn(row[offset], &name, &context, NULL, &context_op)) + continue; + + dc_name = zbx_strpool_intern(name); + dc_context = (NULL != context ? zbx_strpool_intern(context) : NULL); + dc_value = zbx_strpool_intern(row[offset + 1]); + zbx_free(name); + zbx_free(context); + + if (2 == offset) + ZBX_STR2UINT64(hostid, row[1]); + + if (NULL != (pmacro = (zbx_um_t **)zbx_hashset_search(&cache->macros, &pmacroid))) + { + if (SUCCEED == um_macro_is_locked(*pmacro) || hostid != (*pmacro)->hostid || + (*pmacro)->name != dc_name) + { + um_cache_deindex_macro(cache, *pmacro); + } + else + index_macro = 0; + + if (SUCCEED == um_macro_is_locked(*pmacro)) + { + macro = um_macro_dup(*pmacro); + um_macro_release(*pmacro); + *pmacro = macro; + } + + (*pmacro)->hostid = hostid; + + zbx_strpool_release((*pmacro)->name); + if (NULL != (*pmacro)->context) + zbx_strpool_release((*pmacro)->context); + zbx_strpool_release((*pmacro)->value); + } + else + { + macro = (zbx_um_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_t)); + macro->macroid = rowid; + macro->refcount = 1; + pmacro = zbx_hashset_insert(&cache->macros, ¯o, sizeof(macro)); + } + + (*pmacro)->hostid = hostid; + (*pmacro)->name = dc_name; + (*pmacro)->context = dc_context; + (*pmacro)->value = dc_value; + (*pmacro)->type = atoi(row[offset + 2]); + (*pmacro)->context_op = context_op; + + if (0 != index_macro) + um_cache_index_macro(cache, *pmacro); + } + + /* handle removed macros */ + for (; SUCCEED == ret; ret = zbx_dbsync_next(sync, &rowid, &row, &tag)) + { + if (NULL == (pmacro = (zbx_um_t **)zbx_hashset_search(&cache->macros, &pmacroid))) + continue; + + um_cache_deindex_macro(cache, *pmacro); + um_macro_release(*pmacro); + zbx_hashset_remove_direct(&cache->macros, pmacro); + } +} + +static void um_cache_sync_hosts(zbx_um_cache_t *cache, zbx_dbsync_t *sync) +{ + unsigned char tag; + int ret; + zbx_uint64_t rowid, templateid, *phostid = &rowid; + char **row; + zbx_um_host_t *host, **phost; + + while (SUCCEED == (ret = zbx_dbsync_next(sync, &rowid, &row, &tag))) + { + /* removed rows will be always at the end of sync list */ + if (ZBX_DBSYNC_ROW_REMOVE == tag) + break; + + if (NULL != (phost = (zbx_um_host_t **)zbx_hashset_search(&cache->hosts, &phostid))) + { + if (SUCCEED == um_host_is_locked(*phost)) + { + host = um_host_dup(*phost); + um_host_release(*phost); + *phost = host; + } + } + else + { + host = (zbx_um_host_t *)__config_mem_malloc_func(NULL, sizeof(zbx_um_host_t)); + host->hostid = rowid; + host->refcount = 1; + zbx_vector_uint64_create_ext(&host->templateids, __config_mem_malloc_func, + __config_mem_realloc_func, __config_mem_free_func); + + phost = zbx_hashset_insert(&cache->hosts, &host, sizeof(host)); + } + + ZBX_DBROW2UINT64(templateid, row[1]); + + zbx_vector_uint64_append(&(*phost)->templateids, templateid); + } + + /* handle removed host template links */ + for (; SUCCEED == ret; ret = zbx_dbsync_next(sync, &rowid, &row, &tag)) + { + int i; + + if (NULL == (phost = (zbx_um_host_t **)zbx_hashset_search(&cache->hosts, &phostid))) + continue; + + ZBX_DBROW2UINT64(templateid, row[1]); + + for (i = 0; i < (*phost)->templateids.values_num; i++) + { + if ((*phost)->templateids.values[i] == templateid) + { + zbx_vector_uint64_remove_noorder(&(*phost)->templateids, i); + if (0 == (*phost)->templateids.values_num) + { + um_host_release(*phost); + zbx_hashset_remove_direct(&cache->hosts, phost); + } + break; + } + } + } +} + +static zbx_um_cache_t *um_cache_sync(zbx_um_cache_t *cache, zbx_dbsync_t *gmacros, zbx_dbsync_t *hmacros, + zbx_dbsync_t *htmpls) +{ + if (0 == gmacros->rows.values_num && 0 == hmacros->rows.values_num && 0 == htmpls->rows.values_num) + return cache; + + if (1 != cache->refcount) + { + zbx_um_cache_t *clone; + + clone = um_cache_dup(cache); + um_cache_release(cache); + cache = clone; + } + + // WDN + um_cache_sync_macros(cache, gmacros, 1); + um_cache_sync_macros(cache, hmacros, 2); + um_cache_sync_hosts(cache, htmpls); + + return cache; +} + + +// WDN debug +static void um_dbsync_add_hmacro_row(zbx_dbsync_t *sync, unsigned char tag, const char *macroid, const char *hostid, + const char *macro, const char *value, const char *type) +{ + zbx_dbsync_row_t *row; + zbx_uint64_t rowid; + + sync->columns_num = 5; + + ZBX_DBROW2UINT64(rowid, macroid); + row = (zbx_dbsync_row_t *)zbx_malloc(NULL, sizeof(zbx_dbsync_row_t)); + row->rowid = rowid; + row->tag = tag; + + row->row = (char **)zbx_malloc(NULL, sizeof(char *) * sync->columns_num); + row->row[0] = zbx_strdup(NULL, macroid); + row->row[1] = zbx_strdup(NULL, hostid); + row->row[2] = zbx_strdup(NULL, macro); + row->row[3] = zbx_strdup(NULL, value); + row->row[4] = zbx_strdup(NULL, type); + + zbx_vector_ptr_append(&sync->rows, row); + + switch (tag) + { + case ZBX_DBSYNC_ROW_ADD: + sync->add_num++; + break; + case ZBX_DBSYNC_ROW_UPDATE: + sync->update_num++; + break; + case ZBX_DBSYNC_ROW_REMOVE: + sync->remove_num++; + break; + } +} + +// WDN debug +static void um_dbsync_add_gmacro_row(zbx_dbsync_t *sync, unsigned char tag, const char *macroid, + const char *macro, const char *value, const char *type) +{ + zbx_dbsync_row_t *row; + zbx_uint64_t rowid; + + sync->columns_num = 4; + + ZBX_DBROW2UINT64(rowid, macroid); + row = (zbx_dbsync_row_t *)zbx_malloc(NULL, sizeof(zbx_dbsync_row_t)); + row->rowid = rowid; + row->tag = tag; + + row->row = (char **)zbx_malloc(NULL, sizeof(char *) * sync->columns_num); + row->row[0] = zbx_strdup(NULL, macroid); + row->row[1] = zbx_strdup(NULL, macro); + row->row[2] = zbx_strdup(NULL, value); + row->row[3] = zbx_strdup(NULL, type); + + zbx_vector_ptr_append(&sync->rows, row); + + switch (tag) + { + case ZBX_DBSYNC_ROW_ADD: + sync->add_num++; + break; + case ZBX_DBSYNC_ROW_UPDATE: + sync->update_num++; + break; + case ZBX_DBSYNC_ROW_REMOVE: + sync->remove_num++; + break; + } +} + +// WDN +void um_dbsync_clear(zbx_dbsync_t *sync) +{ + /* free the resources allocated by row pre-processing */ + zbx_vector_ptr_clear_ext(&sync->columns, zbx_ptr_free); + zbx_vector_ptr_destroy(&sync->columns); + + zbx_free(sync->row); + + if (ZBX_DBSYNC_UPDATE == sync->mode) + { + int i, j; + zbx_dbsync_row_t *row; + + for (i = 0; i < sync->rows.values_num; i++) + { + row = (zbx_dbsync_row_t *)sync->rows.values[i]; + + if (NULL != row->row) + { + for (j = 0; j < sync->columns_num; j++) + zbx_free(row->row[j]); + + zbx_free(row->row); + } + + zbx_free(row); + } + + zbx_vector_ptr_destroy(&sync->rows); + } + else + { + DBfree_result(sync->dbresult); + sync->dbresult = NULL; + } +} + + +#define HOSTS_MAX 200000 +#define MACROS_MAX 25 +#define VALUES_MAX 2 + +// WDN +static void prepare_hmacro_sync(zbx_dbsync_t *sync, int hosts, int tag, const char *value_prefix) +{ + int i, j, k, id; + char hostid[64], macroid[64], macro[64], value[64]; + + for (i = 0; i < hosts; i++) + { + zbx_snprintf(hostid, sizeof(hostid), "%d", 1000000 + i); + + for (j = 0; j < MACROS_MAX; j++) + { + id = (i * MACROS_MAX * VALUES_MAX) + (j * VALUES_MAX) + 1; + zbx_snprintf(macroid, sizeof(macroid), "%d", id); + zbx_snprintf(macro, sizeof(macro), "{$M%d}", j + 1); + zbx_snprintf(value, sizeof(value), "%s%d", value_prefix, j + 1); + + um_dbsync_add_hmacro_row(sync, tag, macroid, hostid, macro, value, "0"); + + for (k = 1; k < VALUES_MAX; k++) + { + id = (i * MACROS_MAX * VALUES_MAX) + (j * VALUES_MAX) + 1 + k; + zbx_snprintf(macroid, sizeof(macroid), "%d", id); + zbx_snprintf(macro, sizeof(macro), "{$M%d:%d}", (j + 1) , k); + zbx_snprintf(value, sizeof(value), "%s%d+%d", value_prefix, j + 1, k); + + um_dbsync_add_hmacro_row(sync, tag, macroid, hostid, macro, value, "0"); + } + } + } +} + +// WDN +static void prepare_hmacro_sync_update(zbx_dbsync_t *sync, int hosts, int tag, const char *value_prefix) +{ + int i, j, k, id; + char hostid[64], macroid[64], macro[64], value[64]; + + for (i = 0; i < 1; i++) + { + zbx_snprintf(hostid, sizeof(hostid), "%d", 1000000 + i); + + for (j = 0; j < 1; j++) + { + id = (i * MACROS_MAX * VALUES_MAX) + (j * VALUES_MAX) + 1; + zbx_snprintf(macroid, sizeof(macroid), "%d", id); + zbx_snprintf(macro, sizeof(macro), "{$NEW.M%d}", j + 1); + zbx_snprintf(value, sizeof(value), "%s%d", value_prefix, j + 1); + + um_dbsync_add_hmacro_row(sync, tag, macroid, hostid, macro, value, "0"); + + for (k = 1; k < 2; k++) + { + id = (i * MACROS_MAX * VALUES_MAX) + (j * VALUES_MAX) + 1 + k; + zbx_snprintf(macroid, sizeof(macroid), "%d", id); + zbx_snprintf(macro, sizeof(macro), "{$M%d:%d}", (j + 1) , k); + zbx_snprintf(value, sizeof(value), "new.%s%d+%d", value_prefix, j + 1, k); + + um_dbsync_add_hmacro_row(sync, tag, macroid, hostid, macro, value, "0"); + } + } + } +} + +// WDN +static void prepare_hmacro_sync_delete(zbx_dbsync_t *sync, int hosts, int tag, const char *value_prefix) +{ + int i, j, k, id; + char hostid[64], macroid[64], macro[64], value[64]; + + for (i = 0; i < 1; i++) + { + zbx_snprintf(hostid, sizeof(hostid), "%d", 1000000 + i); + + for (j = 0; j < 1; j++) + { + id = (i * MACROS_MAX * VALUES_MAX) + (j * VALUES_MAX) + 1; + zbx_snprintf(macroid, sizeof(macroid), "%d", id); + zbx_snprintf(macro, sizeof(macro), "{$NEW.M%d}", j + 1); + zbx_snprintf(value, sizeof(value), "%s%d", value_prefix, j + 1); + + um_dbsync_add_hmacro_row(sync, tag, macroid, hostid, macro, value, "0"); + + for (k = 1; k < 2; k++) + { + id = (i * MACROS_MAX * VALUES_MAX) + (j * VALUES_MAX) + 1 + k; + zbx_snprintf(macroid, sizeof(macroid), "%d", id); + zbx_snprintf(macro, sizeof(macro), "{$M%d:%d}", (j + 1) , k); + zbx_snprintf(value, sizeof(value), "new.%s%d+%d", value_prefix, j + 1, k); + + um_dbsync_add_hmacro_row(sync, tag, macroid, hostid, macro, value, "0"); + } + } + } +} + +// WDN debug +static void um_dbsync_add_htmpl_row(zbx_dbsync_t *sync, unsigned char tag, const char *hostid, + const char *templateid) +{ + zbx_dbsync_row_t *row; + + sync->columns_num = 2; + + row = (zbx_dbsync_row_t *)zbx_malloc(NULL, sizeof(zbx_dbsync_row_t)); + ZBX_DBROW2UINT64(row->rowid, hostid); + row->tag = tag; + + row->row = (char **)zbx_malloc(NULL, sizeof(char *) * sync->columns_num); + row->row[0] = zbx_strdup(NULL, hostid); + row->row[1] = zbx_strdup(NULL, templateid); + + zbx_vector_ptr_append(&sync->rows, row); + + switch (tag) + { + case ZBX_DBSYNC_ROW_ADD: + sync->add_num++; + break; + case ZBX_DBSYNC_ROW_UPDATE: + sync->update_num++; + break; + case ZBX_DBSYNC_ROW_REMOVE: + sync->remove_num++; + break; + } +} + +static void prepare_htmpl_sync(zbx_dbsync_t *sync) +{ + int i, j, k, id; + char hostid[64], macroid[64], macro[64], value[64]; + + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_ADD, "1000000", "1000001"); + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_ADD, "1000000", "1000002"); + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_ADD, "1000000", "1000003"); + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_ADD, "1000002", "1000004"); + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_ADD, "1000002", "1000005"); + +} + +static void prepare_htmpl_sync_delete(zbx_dbsync_t *sync) +{ + int i, j, k, id; + char hostid[64], macroid[64], macro[64], value[64]; + + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_REMOVE, "1000002", "1000004"); + um_dbsync_add_htmpl_row(sync, ZBX_DBSYNC_ROW_REMOVE, "1000002", "1000005"); +} + +static int macro_compare_by_id(const void *d1, const void *d2) +{ + const zbx_um_t *um1 = *(const zbx_um_t * const *)d1; + const zbx_um_t *um2 = *(const zbx_um_t * const *)d2; + + ZBX_RETURN_IF_DBL_NOT_EQUAL(um1->macroid, um2->macroid); + + return 0; +} + +static int host_compare_by_id(const void *d1, const void *d2) +{ + const zbx_um_host_t *h1 = *(const zbx_um_host_t * const *)d1; + const zbx_um_host_t *h2 = *(const zbx_um_host_t * const *)d2; + + ZBX_RETURN_IF_DBL_NOT_EQUAL(h1->hostid, h2->hostid); + + return 0; +} + +static void um_host_dump(zbx_um_cache_t *cache, const zbx_uint64_t hostid, int indent) +{ + zbx_um_host_t **phost; + int i; + zbx_uint64_t *phostid = &hostid; + + for (i = 0; i < indent; i++) + printf(" "); + + printf("hostid:" ZBX_FS_UI64 ":\n", hostid); + + if (NULL != (phost = (zbx_um_host_t **)zbx_hashset_search(&cache->hosts, &phostid))) + { + for (i = 0; i < (*phost)->templateids.values_num; i++) + um_host_dump(cache, (*phost)->templateids.values[i], indent + 1); + } +} + +// WDN +static void um_cache_dump(zbx_um_cache_t *cache) +{ + zbx_hashset_iter_t iter; + zbx_um_t **pmacro, *macro; + zbx_um_index_t **pindex; + zbx_um_host_t **phost, *host; + zbx_vector_ptr_t list; + int i; + + zbx_vector_ptr_create(&list); + + printf("=== Cache dump ===\n"); + + zbx_hashset_iter_reset(&cache->macros, &iter); + while (NULL != (pmacro = (zbx_um_t **)zbx_hashset_iter_next(&iter))) + zbx_vector_ptr_append(&list, *pmacro); + + zbx_vector_ptr_sort(&list, macro_compare_by_id); + + for (i = 0; i < list.values_num; i++) + { + macro = (zbx_um_t *)list.values[i]; + + printf("macroid:" ZBX_FS_UI64 ": hostid:" ZBX_FS_UI64" %s%s%s=%s\n", macro->macroid, + macro->hostid, macro->name, (NULL == macro->context ? "" : ":"), + ZBX_NULL2EMPTY_STR(macro->context), macro->value); + } + zbx_vector_ptr_clear(&list); + + printf("=== Index dump ===\n"); + zbx_hashset_iter_reset(&cache->index, &iter); + while (NULL != (pindex = (zbx_um_index_t **)zbx_hashset_iter_next(&iter))) + { + printf("hostid:" ZBX_FS_UI64 " macro:%s\n", (*pindex)->hostid, (*pindex)->macro); + + for (i = 0; i < (*pindex)->values.values_num; i++) + zbx_vector_ptr_append(&list, (*pindex)->values.values[i]); + + zbx_vector_ptr_sort(&list, macro_compare_by_id); + + for (i = 0; i < list.values_num; i++) + { + macro = (zbx_um_t *)list.values[i]; + + printf("macroid:" ZBX_FS_UI64 ": hostid:" ZBX_FS_UI64" %s%s%s=%s\n", macro->macroid, + macro->hostid, macro->name, (NULL == macro->context ? "" : ":"), + ZBX_NULL2EMPTY_STR(macro->context), macro->value); + } + zbx_vector_ptr_clear(&list); + } + + printf("=== Host dump ===\n"); + + zbx_hashset_iter_reset(&cache->hosts, &iter); + while (NULL != (phost = (zbx_um_host_t **)zbx_hashset_iter_next(&iter))) + zbx_vector_ptr_append(&list, *phost); + + zbx_vector_ptr_sort(&list, host_compare_by_id); + + for (i = 0; i < list.values_num; i++) + { + host = (zbx_um_host_t *)list.values[i]; + um_host_dump(cache, host->hostid, 0); + } + + printf("=== === ===\n"); + + zbx_vector_ptr_destroy(&list); +} + +// WDN debug +static void snapshot_start(struct timeval *s1) +{ + gettimeofday(s1, NULL); +} + +static int snapshot_end(struct timeval *s1) +{ + struct timeval s2; + int diff; + + gettimeofday(&s2, NULL); + diff = s2.tv_sec - s1->tv_sec; + if (s2.tv_usec < s1->tv_usec) + { + s2.tv_usec += 1000000; + diff--; + } + diff = diff * 1000 + (s2.tv_usec - s1->tv_usec) / 1000; + + return diff; +} + +void test_macros() +{ + zbx_dbsync_t hmacros, gmacros, htmpls, hmacros2, htmpls2; + zbx_um_cache_t *cache1, *cache2; + zbx_hashset_iter_t iter; + char *str; + struct timeval s1; + + um_strpool_init(); + + zbx_dbsync_init(&gmacros, ZBX_DBSYNC_UPDATE); + zbx_dbsync_init(&hmacros, ZBX_DBSYNC_UPDATE); + zbx_dbsync_init(&htmpls, ZBX_DBSYNC_UPDATE); + + zbx_dbsync_init(&hmacros2, ZBX_DBSYNC_UPDATE); + zbx_dbsync_init(&htmpls2, ZBX_DBSYNC_UPDATE); + + prepare_hmacro_sync(&hmacros, HOSTS_MAX, ZBX_DBSYNC_ROW_ADD, "#"); + prepare_hmacro_sync_update(&hmacros2, HOSTS_MAX, ZBX_DBSYNC_ROW_UPDATE, "#"); + //prepare_hmacro_sync_delete(&hmacros2, HOSTS_MAX, ZBX_DBSYNC_ROW_REMOVE, "#"); + + //prepare_htmpl_sync(&htmpls); + //prepare_htmpl_sync_delete(&htmpls2); + + cache1 = um_cache_create(); + + snapshot_start(&s1); + + cache1 = um_cache_sync(cache1, &gmacros, &hmacros, &htmpls); + + printf("initial sync: %.3f\n", (double)snapshot_end(&s1) / 1000); + printf("\tinserted:" ZBX_FS_UI64 " modified:" ZBX_FS_UI64 " deleted:" ZBX_FS_UI64 "\n", + hmacros.add_num, hmacros.update_num, hmacros.remove_num); + + //um_cache_dump(cache1); + + cache1->refcount++; + + snapshot_start(&s1); + cache2 = um_cache_sync(cache1, &gmacros, &hmacros2, &htmpls2); + + printf("update sync: %.3f\n", (double)snapshot_end(&s1) / 1000); + printf("\tinserted:" ZBX_FS_UI64 " modified:" ZBX_FS_UI64 " deleted:" ZBX_FS_UI64 "\n", + hmacros2.add_num, hmacros2.update_num, hmacros2.remove_num); + + + + //um_cache_dump(cache1); + //um_cache_dump(cache2); + + um_cache_release(cache1); + um_cache_release(cache2); + + um_dbsync_clear(&gmacros); + um_dbsync_clear(&hmacros); + um_dbsync_clear(&htmpls); + + um_dbsync_clear(&hmacros2); + um_dbsync_clear(&htmpls2); + + printf("strpool unreleased strings: %d\n", strpool.num_data); + zbx_hashset_iter_reset(&strpool, &iter); + while (NULL != (str = zbx_hashset_iter_next(&iter))) + { + str += REFCOUNT_FIELD_SIZE; + printf(" %s\n", str); + } + + um_strpool_destroy(); + + exit(0); +} + + + + + + diff --git a/src/zabbix_server/server.c b/src/zabbix_server/server.c index a156c93b8e..86e2a4cd8c 100644 --- a/src/zabbix_server/server.c +++ b/src/zabbix_server/server.c @@ -995,6 +995,9 @@ int main(int argc, char **argv) char ch; int opt_c = 0, opt_r = 0; + // WDN + test_macros(); + #if defined(PS_OVERWRITE_ARGV) || defined(PS_PSTAT_ARGV) argv = setproctitle_save_env(argc, argv); #endif