[PATCH 1/2] - add puppet minion host cache file config option - add code to use that so we don't have to parse the host inventory + serials every time we load up a new instance

[Seth Vidal]

---
func/commonconfig.py | 1 +
func/overlord/client.py | 112 ++++++++++++++++++++++++++++++-----------------
2 files changed, 73 insertions(+), 40 deletions(-)

diff --git a/func/commonconfig.py b/func/commonconfig.py
index 93dfdc4..639e721 100644
--- a/func/commonconfig.py
+++ b/func/commonconfig.py
@@ -49,6 +49,7 @@ class OverlordConfig(BaseConfig):
puppet_minions = BoolOption(False)
puppet_inventory = Option('/var/lib/puppet/ssl/ca/inventory.txt')
puppet_signed_certs_dir = Option('/var/lib/puppet/ssl/ca/signed')
+ puppet_minion_cache = Option('/var/lib/func/minion_cache')
puppet_crl = Option('/var/lib/puppet/ssl/ca/ca_crl.pem')
host_down_list = Option('/var/lib/func/hosts_down.lst')
allow_unknown_minions = BoolOption(False)
diff --git a/func/overlord/client.py b/func/overlord/client.py
index 3fd2fc2..8d2accd 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -16,6 +16,7 @@ from func.jobthing import RETAIN_INTERVAL

import sys
import os
+from stat import *
import time
import shlex
import subprocess
@@ -313,7 +314,7 @@ class PuppetMinions(Minions):
just_fqdns=False, groups_backend="conf",
delegate=False, minionmap={},exclude_spec=None,**kwargs):
# local host_inv cache
- self._host_inv = {}
+ self._cached_hosts = set([])
self._revoked_serials = []

Minions.__init__(self, spec, port=port, noglobs=noglobs, verbose=verbose,
@@ -328,8 +329,29 @@ class PuppetMinions(Minions):
#these will be returned
tmp_certs = set()
tmp_hosts = set()
- if not self._host_inv:
- # get all hosts
+
+ # if our cache file of hostnames is newer than both the inventory
+ # and the crl, then just read in that list of hostnames to a set
+ # and skip all of the below
+ if not self._cached_hosts:
+ cacheage = 0
+ invage = 1
+ crlage = 1
+ if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.exists(self.overlord_config.puppet_minion_cache):
+ cacheage = os.stat(self.overlord_config.puppet_minion_cache)[ST_MTIME]
+ if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.exists(self.overlord_config.puppet_inventory):
+ invage = os.stat(self.overlord_config.puppet_inventory)[ST_MTIME]
+ if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.exists(self.overlord_config.puppet_crl):
+ crlage = os.stat(self.overlord_config.puppet_crl)[ST_MTIME]
+ if cacheage >= invage and cacheage >= crlage:
+ self._cached_hosts.update(set(open(self.overlord_config.puppet_minion_cache, 'r').readlines()))
+
+ if not self._cached_hosts:
+ # overview
+ # get all hosts from the puppet inventory
+ # remove all the hosts which are revoked by serial
+ # write out this list as a cache so we can load it quickly
+ # in the future if nothing else has changed.
if os.access(self.overlord_config.puppet_inventory, os.R_OK):
fo = open(self.overlord_config.puppet_inventory, 'r')
host_inv = {}
@@ -356,48 +378,58 @@ class PuppetMinions(Minions):
continue
host_inv[hn] = serial
fo.close()
- self._host_inv = host_inv # store ours
-
- # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
- if self.delegate:
- try:
- each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
- shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
- except IndexError:
- return tmp_hosts,tmp_certs
- else:
- each_gloob = shortest_path[0]
-
- # revoked certs
- self._return_revoked_serials(self.overlord_config.puppet_crl)
- for hostname in self._host_inv.keys():
- if int(self._host_inv[hostname], 16) in self._revoked_serials:
- continue
- pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
- if not os.path.exists(pempath):
- continue
- matched_gloob = False
- if fnmatch.fnmatch(hostname, each_gloob):
- matched_gloob = True
- tmp_hosts.add(hostname)
-
- # if we can't match this gloob and the gloob is not REALLY a glob
- # then toss this at gethostbyname_ex() and see if any of the cname
- # or aliases matches _something_ we know about
- if not matched_gloob and not func_utils.re_glob(each_gloob):
- found_by_alias = False
- aliases = func_utils.get_all_host_aliases(each_gloob)
- for name in aliases:
- if name in self._host_inv and int(self._host_inv[name], 16) not in self._revoked_serials:
- if os.path.exists(self.overlord_config.puppet_signed_certs_dir + '/' + name + '.pem'):
+ # don't include the ones which are of revoked certs
+ self._return_revoked_serials(self.overlord_config.puppet_crl)
+ for hostname in host_inv.keys():
+ if int(host_inv[hostname], 16) in self._revoked_serials:
+ continue
+ pempath = '%s/%s.pem' % (self.overlord_config.puppet_signed_certs_dir, hostname)
+ if not os.path.exists(pempath):
+ continue
+ self._cached_hosts.add(hostname)
+
+ # write out the cache of hosts minus the ones excluded by
+ # revoked serials
+ if os.access(self.overlord_config.puppet_minion_cache, os.W_OK):
+ hostcache = open(self.overlord_config.puppet_minion_cache, 'w')
+ for host in self._cached_hosts:
+ hostcache.write('%s\n' % host)
+ hostcache.close()
+
+
+
+ # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
+ if self.delegate:
+ try:
+ each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
+ shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
+ except IndexError:
+ return tmp_hosts,tmp_certs
+ else:
+ each_gloob = shortest_path[0]
+
+ for hostname in self._cached_hosts:
+ matched_gloob = False
+ if fnmatch.fnmatch(hostname, each_gloob):
+ matched_gloob = True
+ tmp_hosts.add(hostname)
+
+ # if we can't match this gloob and the gloob is not REALLY a glob
+ # then toss this at gethostbyname_ex() and see if any of the cname
+ # or aliases matches _something_ we know about
+ if not matched_gloob and not func_utils.re_glob(each_gloob):
+ found_by_alias = False
+ aliases = func_utils.get_all_host_aliases(each_gloob)
+ for name in aliases:
+ if name in self._cached_hosts:
tmp_hosts.add(name)
found_by_alias = True
break

- if self.overlord_config.allow_unknown_minions and not found_by_alias:
- tmp_hosts.add(each_gloob)
+ if self.overlord_config.allow_unknown_minions and not found_by_alias:
+ tmp_hosts.add(each_gloob)

- # don't return certs path - just hosts
+ # don't return certs path - just hosts

return tmp_hosts,tmp_certs

[Seth Vidal]

On my system of 142 puppet minions a '*' list_minions goes from 1.4-1.5s per load to
0.2s per load
---
func/overlord/client.py | 74 +++++++++++++++++++++++-----------------------
1 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/func/overlord/client.py b/func/overlord/client.py
index 8d2accd..b53dfdb 100644
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -337,14 +337,14 @@ class PuppetMinions(Minions):
cacheage = 0
invage = 1
crlage = 1
- if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.exists(self.overlord_config.puppet_minion_cache):
+ if os.access(self.overlord_config.puppet_minion_cache, os.R_OK) and os.path.exists(self.overlord_config.puppet_minion_cache):
cacheage = os.stat(self.overlord_config.puppet_minion_cache)[ST_MTIME]
- if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.exists(self.overlord_config.puppet_inventory):
+ if os.access(self.overlord_config.puppet_inventory, os.R_OK) and os.path.exists(self.overlord_config.puppet_inventory):
invage = os.stat(self.overlord_config.puppet_inventory)[ST_MTIME]
- if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.exists(self.overlord_config.puppet_crl):
+ if os.access(self.overlord_config.puppet_crl, os.R_OK) and os.path.exists(self.overlord_config.puppet_crl):
crlage = os.stat(self.overlord_config.puppet_crl)[ST_MTIME]
if cacheage >= invage and cacheage >= crlage:
- self._cached_hosts.update(set(open(self.overlord_config.puppet_minion_cache, 'r').readlines()))
+ self._cached_hosts.update(set([ line.strip() for line in open(self.overlord_config.puppet_minion_cache, 'r').readlines()]))

if not self._cached_hosts:
# overview
@@ -390,7 +390,7 @@ class PuppetMinions(Minions):

# write out the cache of hosts minus the ones excluded by
# revoked serials
- if os.access(self.overlord_config.puppet_minion_cache, os.W_OK):
+ if os.access(os.path.dirname(self.overlord_config.puppet_minion_cache), os.W_OK):
hostcache = open(self.overlord_config.puppet_minion_cache, 'w')
for host in self._cached_hosts:
hostcache.write('%s\n' % host)
@@ -398,38 +398,38 @@ class PuppetMinions(Minions):



- # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
- if self.delegate:
- try:
- each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
- shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
- except IndexError:
- return tmp_hosts,tmp_certs
- else:
- each_gloob = shortest_path[0]
-
- for hostname in self._cached_hosts:
- matched_gloob = False
- if fnmatch.fnmatch(hostname, each_gloob):
- matched_gloob = True
- tmp_hosts.add(hostname)
-
- # if we can't match this gloob and the gloob is not REALLY a glob
- # then toss this at gethostbyname_ex() and see if any of the cname
- # or aliases matches _something_ we know about
- if not matched_gloob and not func_utils.re_glob(each_gloob):
- found_by_alias = False
- aliases = func_utils.get_all_host_aliases(each_gloob)
- for name in aliases:
- if name in self._cached_hosts:
- tmp_hosts.add(name)
- found_by_alias = True
- break
-
- if self.overlord_config.allow_unknown_minions and not found_by_alias:
- tmp_hosts.add(each_gloob)
-
- # don't return certs path - just hosts
+ # if call is delegated find the shortest path to the minion and use the sub-overlord's certificate
+ if self.delegate:
+ try:
+ each_gloob = func_utils.get_all_host_aliases(each_gloob)[0]
+ shortest_path = dtools.get_shortest_path(each_gloob, self.minionmap)
+ except IndexError:
+ return tmp_hosts,tmp_certs
+ else:
+ each_gloob = shortest_path[0]
+
+ for hostname in self._cached_hosts:
+ matched_gloob = False
+ if fnmatch.fnmatch(hostname, each_gloob):
+ matched_gloob = True
+ tmp_hosts.add(hostname)
+
+ # if we can't match this gloob and the gloob is not REALLY a glob
+ # then toss this at gethostbyname_ex() and see if any of the cname
+ # or aliases matches _something_ we know about
+ if not matched_gloob and not func_utils.re_glob(each_gloob):
+ found_by_alias = False
+ aliases = func_utils.get_all_host_aliases(each_gloob)
+ for name in aliases:
+ if name in self._cached_hosts:
+ tmp_hosts.add(name)
+ found_by_alias = True
+ break
+
+ if self.overlord_config.allow_unknown_minions and not found_by_alias:
+ tmp_hosts.add(each_gloob)
+
+ # don't return certs path - just hosts

return tmp_hosts,tmp_certs