From da59aba33d7bf4a3d157686e4b614c147f35cad2 Mon Sep 17 00:00:00 2001 From: Janis Freibergs Date: Wed, 8 Sep 2021 16:10:36 +0300 Subject: [PATCH] A......... [ZBX-19902] applied batching of related objects to large sets of graphs; avoided repeated query --- .../classes/api/services/CGraphGeneral.php | 175 ++++++++++-------- 1 file changed, 96 insertions(+), 79 deletions(-) diff --git a/ui/include/classes/api/services/CGraphGeneral.php b/ui/include/classes/api/services/CGraphGeneral.php index 032df898a6..531ef74e7e 100644 --- a/ui/include/classes/api/services/CGraphGeneral.php +++ b/ui/include/classes/api/services/CGraphGeneral.php @@ -320,102 +320,119 @@ abstract class CGraphGeneral extends CApiService { $graphids = array_keys($result); - // adding GraphItems - if ($options['selectGraphItems'] !== null && $options['selectGraphItems'] !== API_OUTPUT_COUNT) { - $gitems = API::GraphItem()->get([ - 'output' => $this->outputExtend($options['selectGraphItems'], ['graphid', 'gitemid']), - 'graphids' => $graphids, - 'nopermissions' => true, - 'preservekeys' => true - ]); - $relationMap = $this->createRelationMap($gitems, 'graphid', 'gitemid'); + do { + $offset = count($graphids) - 10000; - $gitems = $this->unsetExtraFields($gitems, ['graphid', 'gitemid'], $options['selectGraphItems']); - $result = $relationMap->mapMany($result, $gitems, 'gitems'); - } - - // adding HostGroups - if ($options['selectGroups'] !== null && $options['selectGroups'] !== API_OUTPUT_COUNT) { - $relationMap = new CRelationMap(); - // discovered items - $dbRules = DBselect( - 'SELECT gi.graphid,hg.groupid'. - ' FROM graphs_items gi,items i,hosts_groups hg'. - ' WHERE '.dbConditionInt('gi.graphid', $graphids). - ' AND gi.itemid=i.itemid'. - ' AND i.hostid=hg.hostid' - ); - while ($relation = DBfetch($dbRules)) { - $relationMap->addRelation($relation['graphid'], $relation['groupid']); + if ($offset < 0) { + $offset = 0; } - $groups = API::HostGroup()->get([ - 'output' => $options['selectGroups'], - 'groupids' => $relationMap->getRelatedIds(), - 'nopermissions' => true, - 'preservekeys' => true - ]); - $result = $relationMap->mapMany($result, $groups, 'groups'); - } + $graphids_batch = array_slice($graphids, $offset); + array_splice($graphids, $offset); - // adding Hosts - if ($options['selectHosts'] !== null && $options['selectHosts'] !== API_OUTPUT_COUNT) { - $hosts = []; - $relationMap = new CRelationMap(); - // discovered items - $dbRules = DBselect( - 'SELECT gi.graphid,i.hostid'. - ' FROM graphs_items gi,items i'. - ' WHERE '.dbConditionInt('gi.graphid', $graphids). - ' AND gi.itemid=i.itemid' - ); - while ($relation = DBfetch($dbRules)) { - $relationMap->addRelation($relation['graphid'], $relation['hostid']); - } - - $related_ids = $relationMap->getRelatedIds(); - - if ($related_ids) { - $hosts = API::Host()->get([ - 'output' => $options['selectHosts'], - 'hostids' => $related_ids, - 'templated_hosts' => true, + // adding GraphItems + if ($options['selectGraphItems'] !== null && $options['selectGraphItems'] !== API_OUTPUT_COUNT) { + $gitems = API::GraphItem()->get([ + 'output' => $this->outputExtend($options['selectGraphItems'], ['graphid', 'gitemid']), + 'graphids' => $graphids_batch, 'nopermissions' => true, 'preservekeys' => true ]); - } + $relation_map = $this->createRelationMap($gitems, 'graphid', 'gitemid'); - $result = $relationMap->mapMany($result, $hosts, 'hosts'); - } + $gitems = $this->unsetExtraFields($gitems, ['graphid', 'gitemid'], $options['selectGraphItems']); + $result = $relation_map->mapMany($result, $gitems, 'gitems'); + unset($gitems, $relation_map); + } - // adding Templates - if ($options['selectTemplates'] !== null && $options['selectTemplates'] !== API_OUTPUT_COUNT) { - $templates = []; - $relationMap = new CRelationMap(); - // discovered items - $dbRules = DBselect( - 'SELECT gi.graphid,i.hostid'. - ' FROM graphs_items gi,items i'. - ' WHERE '.dbConditionInt('gi.graphid', $graphids). - ' AND gi.itemid=i.itemid' + $select_hostgroups = ($options['selectGroups'] !== null && $options['selectGroups'] !== API_OUTPUT_COUNT); + $select_hosts = ($options['selectHosts'] !== null && $options['selectHosts'] !== API_OUTPUT_COUNT); + $select_templates = ($options['selectTemplates'] !== null + && $options['selectTemplates'] !== API_OUTPUT_COUNT ); - while ($relation = DBfetch($dbRules)) { - $relationMap->addRelation($relation['graphid'], $relation['hostid']); + + if (!($select_hostgroups || $select_hosts || $select_templates)) { + continue; } - $related_ids = $relationMap->getRelatedIds(); + $graphids_where = dbConditionInt('gi.graphid', $graphids_batch); + unset($graphids_batch); - if ($related_ids) { - $templates = API::Template()->get([ - 'output' => $options['selectTemplates'], - 'templateids' => $related_ids, + // adding HostGroups + if ($select_hostgroups) { + $relation_map = new CRelationMap(); + // discovered items + $db_rules = DBselect( + 'SELECT gi.graphid,hg.groupid'. + ' FROM graphs_items gi,items i,hosts_groups hg'. + ' WHERE '.$graphids_where. + ' AND gi.itemid=i.itemid'. + ' AND i.hostid=hg.hostid' + ); + + while ($relation = DBfetch($db_rules)) { + $relation_map->addRelation($relation['graphid'], $relation['groupid']); + } + + $groups = API::HostGroup()->get([ + 'output' => $options['selectGroups'], + 'groupids' => $relation_map->getRelatedIds(), 'nopermissions' => true, 'preservekeys' => true ]); - } + $result = $relation_map->mapMany($result, $groups, 'groups'); + unset($groups, $relation_map); + } + + if ($select_hosts || $select_templates) { + $relation_map = new CRelationMap(); + // discovered items + $db_rules = DBselect( + 'SELECT gi.graphid,i.hostid'. + ' FROM graphs_items gi,items i'. + ' WHERE '.$graphids_where. + ' AND gi.itemid=i.itemid' + ); + unset($graphids_where); - $result = $relationMap->mapMany($result, $templates, 'templates'); - } + while ($relation = DBfetch($db_rules)) { + $relation_map->addRelation($relation['graphid'], $relation['hostid']); + } + + $related_ids = $relation_map->getRelatedIds(); + + if (!$related_ids) { + continue; + } + + // adding Hosts + if ($select_hosts) { + $hosts = API::Host()->get([ + 'output' => $options['selectHosts'], + 'hostids' => $related_ids, + 'templated_hosts' => true, + 'nopermissions' => true, + 'preservekeys' => true + ]); + $result = $relation_map->mapMany($result, $hosts, 'hosts'); + unset($hosts); + } + + // adding Templates + if ($select_templates) { + $templates = API::Template()->get([ + 'output' => $options['selectTemplates'], + 'templateids' => $related_ids, + 'nopermissions' => true, + 'preservekeys' => true + ]); + $result = $relation_map->mapMany($result, $templates, 'templates'); + unset($templates); + } + + unset($relation_map, $related_ids); + } + } while ($offset > 0); return $result; } -- 2.33.0.windows.2