func/minion func/overlord

[Seth Vidal]

func/minion/modules/getfile.py | 64 ++++++++++++++++++++++++
func/overlord/modules/getfile.py | 101 +++++++++++++++++++++++++++++++++++++++
2 files changed, 165 insertions(+)

New commits:
commit 715699573358e4d93b756626fa58278e646630e1
Author: Louis Coilliot <louis.coilliot-***@public.gmane.org>
Date: Wed Dec 22 15:42:33 2010 -0500

add getfile module to allow retrieving arbitrary files from a minion

diff --git a/func/minion/modules/getfile.py b/func/minion/modules/getfile.py
new file mode 100644
index 0000000..b2f23b1
--- /dev/null
+++ b/func/minion/modules/getfile.py
@@ -0,0 +1,64 @@
+#!/usr/bin/python
+#Copyright (C) 2010 Louis-Frederic Coilliot
+#
+#This program is free software: you can redistribute it and/or modify
+#it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Get a file, chunk by chunk. Minion side."""
+import sys
+import sha
+import func_module
+try:
+# py 2.4
+ from base64 import b64encode
+except ImportError:
+# py 2.3
+ from base64 import encodestring as b64encode
+
+
+class GetFile(func_module.FuncModule):
+ """Get a file, chunk by chunk"""
+ def chunkslen(self, filename):
+ """Define the number of chunks of size=bufsize there is in the file"""
+ bufsize = 60000
+ try:
+ fic = open(filename, "r")
+ except IOError, err:
+ sys.stderr.write("Unable to open file: %s: %s\n" % (filename, err))
+ return(-1)
+ chunkslen = 0
+ while True:
+ fic.seek(bufsize*chunkslen)
+ data = fic.read(1024)
+ if not data:
+ break
+ chunkslen += 1
+ fic.close()
+ return(chunkslen)
+
+ def getchunk(self, chunknum, filename):
+ """Get a chunk of the file, after a seek to the right position"""
+ bufsize = 60000
+ try:
+ fic = open(filename, "r")
+ except IOError, err:
+ sys.stderr.write("Unable to open file: %s: %s\n" % (filename, err))
+ checksum = -1
+ return(checksum, '')
+ fic.seek(bufsize*chunknum)
+ chunk = b64encode(fic.read(bufsize))
+ mysha = sha.new()
+ mysha.update(chunk)
+ checksum = mysha.hexdigest()
+ fic.close()
+ return(checksum, chunk)
+
diff --git a/func/overlord/modules/getfile.py b/func/overlord/modules/getfile.py
new file mode 100644
index 0000000..8c4ffd2
--- /dev/null
+++ b/func/overlord/modules/getfile.py
@@ -0,0 +1,101 @@
+#!/usr/bin/python
+#Copyright (C) 2010 Louis-Frederic Coilliot
+#
+#This program is free software: you can redistribute it and/or modify
+#it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Get a file on the minions, chunk by chunk. Overlord side."""
+from func.overlord import overlord_module
+import os
+import sha
+try:
+ # py 2.4
+ from base64 import b64decode
+except ImportError:
+ # py 2.3
+ from base64 import decodestring as b64decode
+
+class getfile(overlord_module.BaseModule):
+ """Get a file on the minions"""
+ def get(self, source, foldertarget):
+ """Get a file on the minions, chunk by chunk. Save the files locally"""
+ chunkslendict = self.parent.run("getfile", "chunkslen", [ source ])
+ # Exclude results for minions with a REMOTE_ERROR
+ chunkslens = [ clen for clen in chunkslendict.values()\
+ if (type(clen) == type(int())) ]
+ maxchunk = max(chunkslens)
+
+ if maxchunk == -1:
+ msg = 'Unable to open the file on the minion(s)'
+ status = 1
+ return status, msg
+
+ if not os.path.isdir(foldertarget):
+ try:
+ os.mkdir(foldertarget)
+ except OSError:
+ msg = 'Problem during the creation of the folder %s'\
+ % (foldertarget)
+ status = 1
+ return status, msg
+
+ if not os.access(foldertarget, os.W_OK):
+ msg = 'The folder %s is not writeable' % (foldertarget)
+ status = 1
+ return status, msg
+
+ nullsha = sha.new().hexdigest()
+ sourcebasename = os.path.basename(source)
+ excluderrlist = []
+
+ for chunknum in range(maxchunk):
+ currentchunks = self.parent.run("getfile", "getchunk",
+ [chunknum, source]).items()
+ for minion, chunkparams in currentchunks:
+ if minion in excluderrlist:
+ # previous error reported
+ continue
+ try:
+ checksum, chunk = chunkparams
+ except ValueError:
+ # Probably a REMOTE_ERROR
+ excluderrlist.append(minion)
+ continue
+ mysha = sha.new()
+ mysha.update(chunk)
+ if checksum == -1:
+ # On this minion there was no file to get
+ continue
+ if mysha.hexdigest() == nullsha:
+ # On this minion there is no more chunk to get
+ continue
+ minionfolder = foldertarget+'/'+minion
+ if mysha.hexdigest() == checksum:
+ if not os.path.isdir(minionfolder):
+ try:
+ os.mkdir(minionfolder)
+ except OSError:
+ excluderrlist.append(minion)
+ continue
+ if chunknum == 0:
+ fic = open(minionfolder+'/'+sourcebasename, 'w')
+ else:
+ fic = open(minionfolder+'/'+sourcebasename, 'a')
+ fic.write(b64decode(chunk))
+ fic.close()
+ else:
+ # Error - checksum failed during copy
+ # Delete the partial file
+ # Abort the copy for this minion
+ excluderrlist.append(minion)
+ os.remove(minionfolder+'/'+sourcebasename)
+ return 0, foldertarget