FAQ

Page Discussion History

Difference between revisions of "MemCache-Preload"

m (Ingredients)
 
(Mount, Preload, Unmount)
 
(8 intermediate revisions by one user not shown)
Line 3: Line 3:
 
'''This setups is only a proof of concept = means: experimental, but works for me.'''
 
'''This setups is only a proof of concept = means: experimental, but works for me.'''
  
# serve the equests
+
# serve the requests
 
#* '''[http://wiki.nginx.org/HttpMemcachedModule nginx]''': webserver with memcache api
 
#* '''[http://wiki.nginx.org/HttpMemcachedModule nginx]''': webserver with memcache api
#* '''[http://www.couchbase.com/downloads membase]''': persistent memcache server
+
#* '''[http://www.couchbase.com/downloads couchbase]''': persistent memcache server
 
# preload memcache
 
# preload memcache
 
#* python: a programming language
 
#* python: a programming language
Line 11: Line 11:
 
#* '''[http://code.google.com/p/fusepy/source/browse/trunk/fuse.py fuse.py]''': python library for fuse
 
#* '''[http://code.google.com/p/fusepy/source/browse/trunk/fuse.py fuse.py]''': python library for fuse
 
#* '''[http://bazaar.launchpad.net/~python-memcached-team/python-memcached/trunk/view/head:/memcache.py memcache.py]''': python library for memcache
 
#* '''[http://bazaar.launchpad.net/~python-memcached-team/python-memcached/trunk/view/head:/memcache.py memcache.py]''': python library for memcache
#* '''memfis.py''': experimental fuse file system
+
#* '''memfis.py''': experimental fuse file system (new: symbolic links)
  
 
= Server Setup =
 
= Server Setup =
Line 39: Line 39:
 
}
 
}
 
</geshi>
 
</geshi>
* to do: error, fallback ...
+
To do:
 +
* full example with error and fallback
 +
* scetch the goal of a complete system with php offloading
  
 
= Preload Memcache =
 
= Preload Memcache =
Line 54: Line 56:
 
== Mount, Preload, Unmount ==
 
== Mount, Preload, Unmount ==
 
<geshi lang=bash>
 
<geshi lang=bash>
memfis.d/memfis.py /mnt/memfis
+
./memfis.py /mnt/memfis
 
cp -a <source>/* /mnt/memfis
 
cp -a <source>/* /mnt/memfis
 
sudo unmount /mnt/memfis
 
sudo unmount /mnt/memfis
Line 64: Line 66:
 
#!/usr/bin/env python
 
#!/usr/bin/env python
 
# -*- coding: utf-8 -*-
 
# -*- coding: utf-8 -*-
# (c) 2011, titusx at gmx.de
+
"""memcache filesystem, v.20120415"""
 +
"""(c) 2011-2012, titusx at gmx.de"""
 +
 
  
#import syslog, binascii
 
 
from time  import time, strftime
 
from time  import time, strftime
 
from sys    import argv, exit
 
from sys    import argv, exit
 
from socket import gethostname
 
from socket import gethostname
from os    import getuid, getgid, O_RDONLY, O_WRONLY, O_RDWR
+
from os    import getuid, getgid
 
from errno  import *
 
from errno  import *
from stat  import S_IFDIR, S_IFREG
+
from stat  import S_IFDIR, S_IFREG, S_IFLNK
from fuse  import FUSE, FuseOSError, Operations, LoggingMixIn
+
from fuse  import FUSE, FuseOSError, Operations
 
from json  import dumps, loads
 
from json  import dumps, loads
 
import memcache as mc
 
import memcache as mc
#import pylibmc as mc
 
  
 +
 +
# a class with preferences an handy functions
 
class Auxiliary(object):
 
class Auxiliary(object):
  
     def ini(self): # custom ini
+
    # custom ini
         self.debugp = False #'/var/log/memfis-debug.log'
+
     def ini(self):
 +
        # debug switch (off="None") and path
 +
         self.debugp = '/tmp/memfis-debug.log'
 +
        # connection to the memcache server
 
         self.server = ["127.0.0.1:11211"]
 
         self.server = ["127.0.0.1:11211"]
         self.memfis = None
+
        # hook for the (open) memcache object
 +
         self.mcache = None
 +
        # don't use "access time"
 
         self.noatim = True
 
         self.noatim = True
 +
        # prefix for each database entry
 
         self.prefix = "memfis://" + gethostname()
 
         self.prefix = "memfis://" + gethostname()
 +
        # prefix of the file counter
 
         self.countr = self.prefix + "/?cntr"
 
         self.countr = self.prefix + "/?cntr"
 +
        # prefix of the validation counter
 
         self.erased = self.prefix + "/?free"
 
         self.erased = self.prefix + "/?free"
 +
        # init time of the filesystem
 
         self.initim = self.now()
 
         self.initim = self.now()
 +
        # owner of the filessystem
 
         self.iniuid = getuid()
 
         self.iniuid = getuid()
 
         self.inigid = getgid()
 
         self.inigid = getgid()
  
     def now(self): # now
+
    # the time function of the file system
 +
     def now(self):
 
         return time()
 
         return time()
  
     def mka(self): # mkattr
+
    # constructs a default attribute
 +
     def mka(self):
 
         Attr = dict(
 
         Attr = dict(
 
             st_ino    = 0,
 
             st_ino    = 0,
             #st_dev    = 0,
+
             #st_dev    = 0, # currently unused
             #st_rdev    = 0,
+
             #st_rdev    = 0, # currently unused
             #st_blksize = 20000000,
+
             st_blksize = 1024, #20000000
             #st_blocks  = 1,
+
             st_blocks  = 1,
             st_nlink  = 2,
+
             st_nlink  = 2, # currently wrong used
 
             st_size    = 4,
 
             st_size    = 4,
 
             st_mode    = 0,
 
             st_mode    = 0,
Line 112: Line 128:
 
         return Attr
 
         return Attr
  
     def dbg(self, mssg): # debug
+
    # writes debug lines into log file
 +
     def dbg(self, dmsg):
 
         if self.debugp:
 
         if self.debugp:
 
             stamp = strftime("%Y%m%d-%H:%M.%S")
 
             stamp = strftime("%Y%m%d-%H:%M.%S")
 
             log = open(self.debugp, 'a')
 
             log = open(self.debugp, 'a')
             log.write("[%s] %s\n" % (stamp, mssg))
+
             log.write("[%s] %s\n" % (stamp, dmsg))
 
             log.close()
 
             log.close()
 
         return
 
         return
  
     def p2a(self, path): # attr
+
    # generate the attribute key from a given path
 +
     def p2a(self, path):
 
         return self.prefix + path + "?attr"
 
         return self.prefix + path + "?attr"
  
     def p2x(self, path): # xattr
+
    # generate the extended key from a given path
 +
     def p2x(self, path):
 
         return self.prefix + path + "?xttr"
 
         return self.prefix + path + "?xttr"
  
     def p2d(self, path): # data
+
    # generate the data key from a given path
 +
     def p2d(self, path):
 
         return self.prefix + path
 
         return self.prefix + path
  
     def c2n(self, cntr): # counter to node
+
    # generate the node key from a given counter
 +
     def c2n(self, cntr):
 
         return self.prefix + "/?node=%08i" % (cntr)
 
         return self.prefix + "/?node=%08i" % (cntr)
  
     def par(self, path): # parent+file
+
    # splits a file path into parent directory and file name
 +
     def par(self, path):
 
         if path == "/":
 
         if path == "/":
 
             return None
 
             return None
Line 140: Line 162:
 
         return Splt
 
         return Splt
  
     def cnt(self, path): # counter
+
    # adds a pointer, increases and returns the counter
         Cntr = self.memfis.incr(self.countr)
+
     def cnt(self, path):
 +
         Cntr = self.mcache.incr(self.countr)
 
         try:
 
         try:
             self.memfis.add(self.c2n(Cntr), path)
+
             self.mcache.add(self.c2n(Cntr), path)
 
         except:
 
         except:
 
             raise FuseOSError(EEXIST)
 
             raise FuseOSError(EEXIST)
 
         return Cntr
 
         return Cntr
  
     def fre(self, cntr): # erased
+
    # increments the counter of erased objects, return pointer
         self.memfis.incr(self.erased)
+
     def fre(self, cntr):
 +
         self.mcache.incr(self.erased)
 
         return self.c2n(cntr)
 
         return self.c2n(cntr)
  
 +
    # encodes a complex object into a string
 
     def enc(self, pobj):
 
     def enc(self, pobj):
 
         return dumps(pobj, sort_keys=True, indent=1)
 
         return dumps(pobj, sort_keys=True, indent=1)
  
 +
    # decodes a string into a complex object
 
     def dec(self, strg):
 
     def dec(self, strg):
 
         return loads(strg)
 
         return loads(strg)
 +
 +
    # adds a node to the file system
 +
    def mkn(self, path, mode, size):
 +
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("addnode:", path, oct(mode), mode))
 +
        JAttr = self.mcache.get(self.p2a(path))
 +
        if JAttr != None:
 +
            raise FuseOSError(EEXIST) # datei existiert
 +
        # a. processing the parent directory
 +
        Splt = self.par(path)
 +
        if Splt != None:
 +
            Attr = self.dec(self.mcache.get(self.p2a(Splt[0])))
 +
            List = self.dec(self.mcache.get(self.p2d(Splt[0])))
 +
            List.append(Splt[1])
 +
            List.sort()
 +
            Attr['st_size']  = len(self.enc(List))
 +
            Attr['st_mtime'] = self.now()
 +
            Attr['st_atime'] = Attr['st_mtime']
 +
            if S_IFDIR & mode:
 +
                Attr['st_nlink'] += 1
 +
            self.mcache.set(self.p2a(Splt[0]), self.enc(Attr))
 +
            self.mcache.set(self.p2d(Splt[0]), self.enc(List))
 +
        # b. processing the new node itself
 +
        Attr = self.mka()
 +
        Attr['st_ino']  = self.cnt(path) # new inode
 +
        Attr['st_uid']  = self.iniuid
 +
        Attr['st_gid']  = self.inigid
 +
        Attr['st_ctime'] = self.now()
 +
        Attr['st_mtime'] = Attr['st_ctime']
 +
        Attr['st_atime'] = Attr['st_ctime']
 +
        Attr['st_mode']  = mode
 +
        Attr['st_size']  = size
 +
        if S_IFDIR & mode:
 +
            Attr['st_nlink'] = 2
 +
        else:
 +
            Attr['st_nlink'] = 1
 +
        self.mcache.set(self.p2a(path), self.enc(Attr))
 +
        return Attr['st_ino']
 +
 +
    # deletes a node from the file system
 +
    def rmn(self, path, dire):
 +
        self.dbg("%-9s %s" % ("delnode:", path))
 +
        if path == "/":
 +
            raise FuseOSError(EPERM) # nicht berechtigt
 +
        JAttr = self.mcache.get(self.p2a(path))
 +
        if JAttr == None:
 +
            raise FuseOSError(ENOENT) # daten nicht gefunden
 +
        else:
 +
            Attr = self.dec(JAttr)
 +
        if dire == True and Attr['st_nlink'] > 2:
 +
            raise FuseOSError(ENOTEMPTY) # verzeichnis nicht leer
 +
        if dire == False and Attr['st_nlink'] != 1:
 +
            raise FuseOSError(ENOSYS) # nicht implementiert
 +
        # deleting attribues, data and inode pointer
 +
        self.mcache.delete_multi([self.p2a(path), self.p2d(path), self.fre(Attr['st_ino'])])
 +
        Splt = self.par(path)
 +
        if Splt != None:
 +
            Attr = self.dec(self.mcache.get(self.p2a(Splt[0])))
 +
            List = self.dec(self.mcache.get(self.p2d(Splt[0])))
 +
            # removing entry from parent directory
 +
            List.remove(Splt[1])
 +
            Attr['st_size']  = len(self.enc(List))
 +
            Attr['st_mtime'] = self.now()
 +
            Attr['st_atime'] = Attr['st_mtime']
 +
            if dire == True:
 +
                Attr['st_nlink'] -= 1
 +
            self.mcache.set(self.p2a(Splt[0]), self.enc(Attr))
 +
            self.mcache.set(self.p2d(Splt[0]), self.enc(List))
  
  
 +
# the fuse file system class
 
class MemFiS(Operations, Auxiliary):
 
class MemFiS(Operations, Auxiliary):
  
     def __init__(self, *args, **kw):
+
     # initialises the file system
 +
    #def __init__(self, *args, **kw):
 +
    def init(self, path):
 +
        # load the preferences (incl. debug)
 
         self.ini()
 
         self.ini()
 +
        # generate a log entry
 
         self.dbg("__INIT__")
 
         self.dbg("__INIT__")
         self.memfis = mc.Client(self.server)
+
        # connect to a server and store the hook
         if self.debugp:
+
         self.mcache = mc.Client(self.server)
            self.memfis.flush_all()
+
         # ONLY FOR TESTING - REMOVE FOR PRODUCTION
         Cntr = self.memfis.get(self.countr)
+
        # deletes the whole database at startup
 +
        #if self.debugp:
 +
        #    self.mcache.flush_all()
 +
        # start a new file system if nothin exitst
 +
         Cntr = self.mcache.get(self.countr)
 
         if Cntr == None:
 
         if Cntr == None:
             self.memfis.set(self.countr, 0)
+
             self.mcache.set(self.countr, 0)
             self.memfis.set(self.erased, 0)
+
             self.mcache.set(self.erased, 0)
         Attr = self.memfis.get(self.p2a("/"))
+
         Attr = self.mcache.get(self.p2a("/"))
 
         if Attr == None:
 
         if Attr == None:
 
             self.mkdir("/", 493)
 
             self.mkdir("/", 493)
  
     def __del__(self):
+
     #def __del__(self):
 +
    def destroy(self, path):
 
         self.dbg("__DEL__")
 
         self.dbg("__DEL__")
         if self.debugp:
+
         # ONLY FOR TESTING - REMOVE FOR PRODUCTION
            self.memfis.flush_all()
+
        # delete the whole database at shutdown
         self.memfis.disconnect_all()
+
        #if self.debugp:
 +
        #    self.mcache.flush_all()
 +
        # disconnect from the server
 +
         self.mcache.disconnect_all()
 +
 
 +
    def access(self, path, mode):
 +
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("ACCESS:", path, oct(mode), mode))
 +
        return 0
  
 
     def chmod(self, path, mode):
 
     def chmod(self, path, mode):
         self.dbg("CHOWN:    %s (mode: '%s'=%i)" % (path, oct(mode), mode))
+
         self.dbg("%-9s %s (mode: '%s'=%i)" % ("CHMOD:", path, oct(mode), mode))
         JAttr = self.memfis.get(self.p2a(path))
+
        # load the attributes
 +
         JAttr = self.mcache.get(self.p2a(path))
 +
        # decode attributes if exists
 
         if JAttr == None:
 
         if JAttr == None:
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
         else:
 
         else:
 
             Attr = self.dec(JAttr)
 
             Attr = self.dec(JAttr)
         if Attr['st_mode'] == mode:
+
        # adjust changed mode and save it
             return 0
+
         if Attr['st_mode'] != mode:
        Attr['st_mode'] = mode
+
             Attr['st_mode'] = mode
        Attr['st_mtime'] = self.now()
+
            Attr['st_mtime'] = self.now()
        self.memfis.set(self.p2a(path), self.enc(Attr))
+
            self.mcache.set(self.p2a(path), self.enc(Attr))
  
 
     def chown(self, path, usid, grid):
 
     def chown(self, path, usid, grid):
         self.dbg("CHMOD:    %s (uid: %i, gid: %i)" % (path, usid, grid))
+
         self.dbg("%-9s %s (uid: %i, gid: %i)" % ("CHOWN:", path, usid, grid))
         JAttr = self.memfis.get(self.p2a(path))
+
         JAttr = self.mcache.get(self.p2a(path))
 
         if JAttr == None:
 
         if JAttr == None:
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
Line 202: Line 314:
 
             Attr = self.dec(JAttr)
 
             Attr = self.dec(JAttr)
 
         Dirt = False
 
         Dirt = False
 +
        # adjust user id
 
         if Attr['st_uid'] != usid and usid >= 0:
 
         if Attr['st_uid'] != usid and usid >= 0:
 
             Attr['st_uid'] = usid
 
             Attr['st_uid'] = usid
 
             Dirt = True
 
             Dirt = True
 +
        # adjust group id
 
         if Attr['st_gid'] != grid and grid >= 0:
 
         if Attr['st_gid'] != grid and grid >= 0:
 
             Attr['st_gid'] = grid
 
             Attr['st_gid'] = grid
 
             Dirt = True
 
             Dirt = True
         if Dirt == False:
+
         # save changed attributes
            return 0
+
         if Dirt != False:
        Attr['st_mtime'] = self.now()
+
        self.memfis.set(self.p2a(path), self.enc(Attr))
+
 
+
    def addnode(self, path, mode, dire):
+
        self.dbg("addnode:  %s (mode: '%s'=%i)" % (path, oct(mode), mode))
+
        JAttr = self.memfis.get(self.p2a(path))
+
        if JAttr != None:
+
            raise FuseOSError(EEXIST) # datei existiert
+
        Splt = self.par(path)
+
         if Splt != None:
+
            Attr = self.dec(self.memfis.get(self.p2a(Splt[0])))
+
            List = self.dec(self.memfis.get(self.p2d(Splt[0])))
+
            List.append(Splt[1])
+
            List.sort()
+
            Attr['st_size']  = len(self.enc(List))
+
 
             Attr['st_mtime'] = self.now()
 
             Attr['st_mtime'] = self.now()
            Attr['st_atime'] = Attr['st_mtime']
+
             self.mcache.set(self.p2a(path), self.enc(Attr))
            if dire == True:
+
                Attr['st_nlink'] += 1
+
             self.memfis.set(self.p2a(Splt[0]), self.enc(Attr))
+
            self.memfis.set(self.p2d(Splt[0]), self.enc(List))
+
        Attr  = self.mka()
+
        Attr['st_ino']  = self.cnt(path)
+
        Attr['st_uid']  = self.iniuid
+
        Attr['st_gid']  = self.inigid
+
        Attr['st_ctime'] = self.now()
+
        Attr['st_mtime'] = Attr['st_ctime']
+
        Attr['st_atime'] = Attr['st_ctime']
+
        if dire == True:
+
            Attr['st_nlink'] = 2
+
            Attr['st_mode']  = S_IFDIR | mode
+
            Attr['st_size']  = len(self.enc(['.', '..']))
+
        else:
+
            Attr['st_nlink'] = 1
+
            Attr['st_mode']  = S_IFREG | mode
+
            Attr['st_size']  = 0
+
        self.memfis.set(self.p2a(path), self.enc(Attr))
+
        return Attr['st_ino']
+
  
     def create(self, path, mode, fi=None):
+
    # creates an inode with file attributes
         self.dbg("CREATE:  %s (mode: '%s'=%i)" % (path, oct(mode), mode))
+
     def create(self, path, mode):
         return self.addnode(path, mode, False)
+
         self.dbg("%-9s %s (mode: '%s'=%i)" % ("CREATE:", path, oct(mode), mode))
 +
         return self.mkn(path, S_IFREG|mode, 0)
  
 +
    # loads the attributes object
 
     def getattr(self, path, fhdl=None):
 
     def getattr(self, path, fhdl=None):
         self.dbg("GETATTR: " + path)
+
         self.dbg("%-9s %s" % ("GETATTR:", path))
         JAttr = self.memfis.get(self.p2a(path))
+
         JAttr = self.mcache.get(self.p2a(path))
 
         if JAttr == None:
 
         if JAttr == None:
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
Line 261: Line 341:
 
             return self.dec(JAttr)
 
             return self.dec(JAttr)
  
 +
    # creates a new directory
 
     def mkdir(self, path, mode):
 
     def mkdir(self, path, mode):
         self.dbg("MKDIR:    %s (mode: '%s'=%i)" % (path, oct(mode), mode))
+
         self.dbg("%-9s %s (mode: '%s'=%i)" % ("MKDIR:", path, oct(mode), mode))
         self.addnode(path, mode, True)
+
        Data = self.enc(['.', '..'])
         self.memfis.set(self.p2d(path), self.enc(['.', '..']))
+
         self.mkn(path, S_IFDIR|mode, len(Data))
 +
         self.mcache.set(self.p2d(path), Data)
  
 
     def read(self, path, size, oset, fhdl):
 
     def read(self, path, size, oset, fhdl):
         self.dbg("READ:    %s (size: %i, offset: %i)" % (path, size, oset))
+
         self.dbg("%-9s %s (size: %i, offset: %i)" % ("READ:", path, size, oset))
 
         if oset != 0:
 
         if oset != 0:
 +
            # to do
 
             raise FuseOSError(ENOSYS) # nicht implementiert
 
             raise FuseOSError(ENOSYS) # nicht implementiert
         JAttr = self.memfis.get(self.p2a(path))
+
         JAttr = self.mcache.get(self.p2a(path))
 
         if JAttr == None:
 
         if JAttr == None:
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
         else:
 
         else:
 
             Attr = self.dec(JAttr)
 
             Attr = self.dec(JAttr)
 +
        #if Attr['st_size'] != size:
 +
        #    raise FuseOSError(EINVAL) # ungültiges argument
 
         if self.noatim == False:
 
         if self.noatim == False:
             Attr['st_atime'] = now()
+
             Attr['st_atime'] = self.now()
             self.memfis.set(self.p2a(path), self.enc(Attr))
+
             self.mcache.set(self.p2a(path), self.enc(Attr))
         Data = self.memfis.get(self.p2d(path))
+
         return self.mcache.get(self.p2d(path))
        return Data
+
  
 
     def readdir(self, path, fhdl):
 
     def readdir(self, path, fhdl):
         self.dbg("READDIR: " + path)
+
         self.dbg("%-9s %s" % ("READDIR:", path))
 
         List = []
 
         List = []
 
         for e in self.dec(self.read(path, 0, 0, fhdl)):
 
         for e in self.dec(self.read(path, 0, 0, fhdl)):
Line 288: Line 372:
 
         return List
 
         return List
  
     def delnode(self, path, dire):
+
     def readlink(self, path):
         self.dbg("delnode: " + path)
+
        self.dbg("%-9s %s" % ("READLINK:", path))
         if path == "/":
+
        return self.mcache.get(self.p2d(path))
            raise FuseOSError(ENOPERM)
+
 
         JAttr = self.memfis.get(self.p2a(path))
+
    def rmdir(self, path):
 +
         self.dbg("%-9s %s" % ("RMDIR:", path))
 +
         self.rmn(path, True)
 +
 
 +
    def symlink(self, path, orig):
 +
        self.dbg("%-9s %s (path: %s)" % ("SYMLINK:", path, orig))
 +
        self.mkn(path, S_IFLNK|511, len(orig))
 +
        self.mcache.set(self.p2d(path), orig)
 +
 
 +
    # adjusts file size
 +
    def truncate(self, path, leng, fhdl):
 +
        self.dbg("%-9s %s (len: %i)" % ("TRUNCATE:", path, leng))
 +
         JAttr = self.mcache.get(self.p2a(path))
 
         if JAttr == None:
 
         if JAttr == None:
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
         else:
 
         else:
 
             Attr = self.dec(JAttr)
 
             Attr = self.dec(JAttr)
        if dire == True and Attr['st_nlink'] > 2:
+
             Attr['st_size'] = leng
             raise FuseOSError(ENOTEMPTY) # verzeichnis nicht leer
+
             self.mcache.replace(self.p2a(path), self.enc(Attr))
        if dire == False and Attr['st_nlink'] != 1:
+
             raise FuseOSError(ENOSYS) # nicht implementiert
+
        self.memfis.delete_multi([self.p2a(path), self.p2d(path), self.fre(Attr['st_ino'])])
+
        Splt = self.par(path)
+
        if Splt != None:
+
            Attr = self.dec(self.memfis.get(self.p2a(Splt[0])))
+
            List = self.dec(self.memfis.get(self.p2d(Splt[0])))
+
            List.remove(Splt[1])
+
            Attr['st_size']  = len(self.enc(List))
+
            Attr['st_mtime'] = self.now()
+
            Attr['st_atime'] = Attr['st_mtime']
+
            if dire == True:
+
                Attr['st_nlink'] -= 1
+
            self.memfis.set(self.p2a(Splt[0]), self.enc(Attr))
+
            self.memfis.set(self.p2d(Splt[0]), self.enc(List))
+
 
+
    def rmdir(self, path):
+
        self.dbg("RMDIR:    " + path)
+
        self.delnode(path, True)
+
  
     def write(self, path, buff, oset, fdhl=None):
+
     def write(self, path, buff, oset, fhdl):
         self.dbg("WRITE:    %s (buffer: %i, offset: %i)" % (path, len(buff), oset))
+
         self.dbg("%-9s %s (buffer: %i, offset: %i)" % ("WRITE:", path, len(buff), oset))
         JAttr = self.memfis.get(self.p2a(path))
+
         JAttr = self.mcache.get(self.p2a(path))
 
         if JAttr == None:
 
         if JAttr == None:
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
             raise FuseOSError(ENOENT) # daten nicht gefunden
 
         else:
 
         else:
 
             Attr = self.dec(JAttr)
 
             Attr = self.dec(JAttr)
 +
        #if oset != Attr['st_size']:
 +
        #    raise FuseOSError(ESPIPE) # unzulaessige suche
 
         if Attr['st_size'] + len(buff) > 20000000:
 
         if Attr['st_size'] + len(buff) > 20000000:
 
             raise FuseOSError(EFBIG) # datei zu gross
 
             raise FuseOSError(EFBIG) # datei zu gross
Line 331: Line 410:
 
         Attr['st_mtime'] = self.now()
 
         Attr['st_mtime'] = self.now()
 
         Attr['st_atime'] = Attr['st_mtime']
 
         Attr['st_atime'] = Attr['st_mtime']
         self.memfis.replace(self.p2a(path), self.enc(Attr))
+
         self.mcache.replace(self.p2a(path), self.enc(Attr))
 
         if oset == 0:
 
         if oset == 0:
             self.memfis.add(self.p2d(path), buff)
+
             self.mcache.add(self.p2d(path), buff)
 
         else:
 
         else:
             self.memfis.append(self.p2d(path), buff)
+
             self.mcache.append(self.p2d(path), buff)
 
         return len(buff)
 
         return len(buff)
  
 
     def unlink(self, path):
 
     def unlink(self, path):
         self.dbg("UNLINK:   " + path)
+
         self.dbg("%-9s %s" % ("UNLINK:", path))
         self.delnode(path, False)
+
         self.rmn(path, False)
 +
 
  
 
if __name__ == '__main__':
 
if __name__ == '__main__':

Latest revision as of 09:47, 27 April 2013

Contents

Ingredients

This setups is only a proof of concept = means: experimental, but works for me.

  1. serve the requests
    • nginx: webserver with memcache api
    • couchbase: persistent memcache server
  2. preload memcache
    • python: a programming language
    • fuse: file system in user space
    • fuse.py: python library for fuse
    • memcache.py: python library for memcache
    • memfis.py: experimental fuse file system (new: symbolic links)

Server Setup

Installation

apt-get install nginx-full
cp /etc/nginx/sites-available/default /etc/nginx/sites-available/default.dpkg-dist
vi /etc/nginx/sites-available/default
dpkg -i membase-server-community_<arch>_<version>.deb
# Browser: http://<hostname>:8091

/etc/nginx/sites-available/default

server {
        listen          80;
        server_name     <webserver>;
        root            /var/www/;
 
        location / {
                index                   index.html;
                default_type            text/plain;
                set $memcached_key      memfis://<hostname>$uri;
                memcached_pass          127.0.0.1:11211;
        }
}

To do:

  • full example with error and fallback
  • scetch the goal of a complete system with php offloading

Preload Memcache

Preparation

mkdir /mnt/memfis
mkdir memfis.d
cd memfis.d
vi memfis.py fuse.py memcache.py
chmod +x memfis.py

Mount, Preload, Unmount

./memfis.py /mnt/memfis
cp -a <source>/* /mnt/memfis
sudo unmount /mnt/memfis

MemFiS Filesystem

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""memcache filesystem, v.20120415"""
"""(c) 2011-2012, titusx at gmx.de"""
 
 
from time   import time, strftime
from sys    import argv, exit
from socket import gethostname
from os     import getuid, getgid
from errno  import *
from stat   import S_IFDIR, S_IFREG, S_IFLNK
from fuse   import FUSE, FuseOSError, Operations
from json   import dumps, loads
import memcache as mc
 
 
# a class with preferences an handy functions
class Auxiliary(object):
 
    # custom ini
    def ini(self):
        # debug switch (off="None") and path
        self.debugp = '/tmp/memfis-debug.log'
        # connection to the memcache server
        self.server = ["127.0.0.1:11211"]
        # hook for the (open) memcache object
        self.mcache = None
        # don't use "access time"
        self.noatim = True
        # prefix for each database entry
        self.prefix = "memfis://" + gethostname()
        # prefix of the file counter
        self.countr = self.prefix + "/?cntr"
        # prefix of the validation counter
        self.erased = self.prefix + "/?free"
        # init time of the filesystem
        self.initim = self.now()
        # owner of the filessystem
        self.iniuid = getuid()
        self.inigid = getgid()
 
    # the time function of the file system
    def now(self):
        return time()
 
    # constructs a default attribute
    def mka(self):
        Attr = dict(
            st_ino     = 0,
            #st_dev     = 0, # currently unused
            #st_rdev    = 0, # currently unused
            st_blksize = 1024, #20000000
            st_blocks  = 1,
            st_nlink   = 2, # currently wrong used
            st_size    = 4,
            st_mode    = 0,
            st_uid     = self.iniuid,
            st_gid     = self.inigid,
            st_atime   = self.initim,
            st_mtime   = self.initim,
            st_ctime   = self.initim )
        return Attr
 
    # writes debug lines into log file
    def dbg(self, dmsg):
        if self.debugp:
            stamp = strftime("%Y%m%d-%H:%M.%S")
            log = open(self.debugp, 'a')
            log.write("[%s] %s\n" % (stamp, dmsg))
            log.close()
        return
 
    # generate the attribute key from a given path
    def p2a(self, path):
        return self.prefix + path + "?attr"
 
    # generate the extended key from a given path
    def p2x(self, path):
        return self.prefix + path + "?xttr"
 
    # generate the data key from a given path
    def p2d(self, path):
        return self.prefix + path
 
    # generate the node key from a given counter
    def c2n(self, cntr):
        return self.prefix + "/?node=%08i" % (cntr)
 
    # splits a file path into parent directory and file name
    def par(self, path):
        if path == "/":
            return None
        Splt = path.rsplit("/",1)
        if Splt[0] == "":
            Splt[0] = "/"
        return Splt
 
    # adds a pointer, increases and returns the counter
    def cnt(self, path):
        Cntr = self.mcache.incr(self.countr)
        try:
            self.mcache.add(self.c2n(Cntr), path)
        except:
            raise FuseOSError(EEXIST)
        return Cntr
 
    # increments the counter of erased objects, return pointer
    def fre(self, cntr):
        self.mcache.incr(self.erased)
        return self.c2n(cntr)
 
    # encodes a complex object into a string
    def enc(self, pobj):
        return dumps(pobj, sort_keys=True, indent=1)
 
    # decodes a string into a complex object
    def dec(self, strg):
        return loads(strg)
 
    # adds a node to the file system
    def mkn(self, path, mode, size):
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("addnode:", path, oct(mode), mode))
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr != None:
            raise FuseOSError(EEXIST) # datei existiert
        # a. processing the parent directory
        Splt = self.par(path)
        if Splt != None:
            Attr = self.dec(self.mcache.get(self.p2a(Splt[0])))
            List = self.dec(self.mcache.get(self.p2d(Splt[0])))
            List.append(Splt[1])
            List.sort()
            Attr['st_size']  = len(self.enc(List))
            Attr['st_mtime'] = self.now()
            Attr['st_atime'] = Attr['st_mtime']
            if S_IFDIR & mode:
                Attr['st_nlink'] += 1
            self.mcache.set(self.p2a(Splt[0]), self.enc(Attr))
            self.mcache.set(self.p2d(Splt[0]), self.enc(List))
        # b. processing the new node itself
        Attr = self.mka()
        Attr['st_ino']   = self.cnt(path) # new inode
        Attr['st_uid']   = self.iniuid
        Attr['st_gid']   = self.inigid
        Attr['st_ctime'] = self.now()
        Attr['st_mtime'] = Attr['st_ctime']
        Attr['st_atime'] = Attr['st_ctime']
        Attr['st_mode']  = mode
        Attr['st_size']  = size
        if S_IFDIR & mode:
            Attr['st_nlink'] = 2
        else:
            Attr['st_nlink'] = 1
        self.mcache.set(self.p2a(path), self.enc(Attr))
        return Attr['st_ino']
 
    # deletes a node from the file system
    def rmn(self, path, dire):
        self.dbg("%-9s %s" % ("delnode:", path))
        if path == "/":
            raise FuseOSError(EPERM) # nicht berechtigt
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            Attr = self.dec(JAttr)
        if dire == True and Attr['st_nlink'] > 2:
            raise FuseOSError(ENOTEMPTY) # verzeichnis nicht leer
        if dire == False and Attr['st_nlink'] != 1:
            raise FuseOSError(ENOSYS) # nicht implementiert
        # deleting attribues, data and inode pointer
        self.mcache.delete_multi([self.p2a(path), self.p2d(path), self.fre(Attr['st_ino'])])
        Splt = self.par(path)
        if Splt != None:
            Attr = self.dec(self.mcache.get(self.p2a(Splt[0])))
            List = self.dec(self.mcache.get(self.p2d(Splt[0])))
            # removing entry from parent directory
            List.remove(Splt[1])
            Attr['st_size']  = len(self.enc(List))
            Attr['st_mtime'] = self.now()
            Attr['st_atime'] = Attr['st_mtime']
            if dire == True:
                Attr['st_nlink'] -= 1
            self.mcache.set(self.p2a(Splt[0]), self.enc(Attr))
            self.mcache.set(self.p2d(Splt[0]), self.enc(List))
 
 
# the fuse file system class
class MemFiS(Operations, Auxiliary):
 
    # initialises the file system
    #def __init__(self, *args, **kw):
    def init(self, path):
        # load the preferences (incl. debug)
        self.ini()
        # generate a log entry
        self.dbg("__INIT__")
        # connect to a server and store the hook
        self.mcache = mc.Client(self.server)
        # ONLY FOR TESTING - REMOVE FOR PRODUCTION
        # deletes the whole database at startup
        #if self.debugp:
        #    self.mcache.flush_all()
        # start a new file system if nothin exitst
        Cntr = self.mcache.get(self.countr)
        if Cntr == None:
            self.mcache.set(self.countr, 0)
            self.mcache.set(self.erased, 0)
        Attr = self.mcache.get(self.p2a("/"))
        if Attr == None:
            self.mkdir("/", 493)
 
    #def __del__(self):
    def destroy(self, path):
        self.dbg("__DEL__")
        # ONLY FOR TESTING - REMOVE FOR PRODUCTION
        # delete the whole database at shutdown
        #if self.debugp:
        #    self.mcache.flush_all()
        # disconnect from the server
        self.mcache.disconnect_all()
 
    def access(self, path, mode):
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("ACCESS:", path, oct(mode), mode))
        return 0
 
    def chmod(self, path, mode):
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("CHMOD:", path, oct(mode), mode))
        # load the attributes
        JAttr = self.mcache.get(self.p2a(path))
        # decode attributes if exists
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            Attr = self.dec(JAttr)
        # adjust changed mode and save it
        if Attr['st_mode'] != mode:
            Attr['st_mode'] = mode
            Attr['st_mtime'] = self.now()
            self.mcache.set(self.p2a(path), self.enc(Attr))
 
    def chown(self, path, usid, grid):
        self.dbg("%-9s %s (uid: %i, gid: %i)" % ("CHOWN:", path, usid, grid))
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            Attr = self.dec(JAttr)
        Dirt = False
        # adjust user id
        if Attr['st_uid'] != usid and usid >= 0:
            Attr['st_uid'] = usid
            Dirt = True
        # adjust group id
        if Attr['st_gid'] != grid and grid >= 0:
            Attr['st_gid'] = grid
            Dirt = True
        # save changed attributes
        if Dirt != False:
            Attr['st_mtime'] = self.now()
            self.mcache.set(self.p2a(path), self.enc(Attr))
 
    # creates an inode with file attributes
    def create(self, path, mode):
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("CREATE:", path, oct(mode), mode))
        return self.mkn(path, S_IFREG|mode, 0)
 
    # loads the attributes object
    def getattr(self, path, fhdl=None):
        self.dbg("%-9s %s" % ("GETATTR:", path))
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            return self.dec(JAttr)
 
    # creates a new directory
    def mkdir(self, path, mode):
        self.dbg("%-9s %s (mode: '%s'=%i)" % ("MKDIR:", path, oct(mode), mode))
        Data = self.enc(['.', '..'])
        self.mkn(path, S_IFDIR|mode, len(Data))
        self.mcache.set(self.p2d(path), Data)
 
    def read(self, path, size, oset, fhdl):
        self.dbg("%-9s %s (size: %i, offset: %i)" % ("READ:", path, size, oset))
        if oset != 0:
            # to do
            raise FuseOSError(ENOSYS) # nicht implementiert
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            Attr = self.dec(JAttr)
        #if Attr['st_size'] != size:
        #    raise FuseOSError(EINVAL) # ungültiges argument
        if self.noatim == False:
            Attr['st_atime'] = self.now()
            self.mcache.set(self.p2a(path), self.enc(Attr))
        return self.mcache.get(self.p2d(path))
 
    def readdir(self, path, fhdl):
        self.dbg("%-9s %s" % ("READDIR:", path))
        List = []
        for e in self.dec(self.read(path, 0, 0, fhdl)):
            List.append( e.encode('ascii') )
        return List
 
    def readlink(self, path):
        self.dbg("%-9s %s" % ("READLINK:", path))
        return self.mcache.get(self.p2d(path))
 
    def rmdir(self, path):
        self.dbg("%-9s %s" % ("RMDIR:", path))
        self.rmn(path, True)
 
    def symlink(self, path, orig):
        self.dbg("%-9s %s (path: %s)" % ("SYMLINK:", path, orig))
        self.mkn(path, S_IFLNK|511, len(orig))
        self.mcache.set(self.p2d(path), orig)
 
    # adjusts file size
    def truncate(self, path, leng, fhdl):
        self.dbg("%-9s %s (len: %i)" % ("TRUNCATE:", path, leng))
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            Attr = self.dec(JAttr)
            Attr['st_size'] = leng
            self.mcache.replace(self.p2a(path), self.enc(Attr))
 
    def write(self, path, buff, oset, fhdl):
        self.dbg("%-9s %s (buffer: %i, offset: %i)" % ("WRITE:", path, len(buff), oset))
        JAttr = self.mcache.get(self.p2a(path))
        if JAttr == None:
            raise FuseOSError(ENOENT) # daten nicht gefunden
        else:
            Attr = self.dec(JAttr)
        #if oset != Attr['st_size']:
        #    raise FuseOSError(ESPIPE) # unzulaessige suche
        if Attr['st_size'] + len(buff) > 20000000:
            raise FuseOSError(EFBIG) # datei zu gross
        Attr['st_size'] += len(buff)
        Attr['st_mtime'] = self.now()
        Attr['st_atime'] = Attr['st_mtime']
        self.mcache.replace(self.p2a(path), self.enc(Attr))
        if oset == 0:
            self.mcache.add(self.p2d(path), buff)
        else:
            self.mcache.append(self.p2d(path), buff)
        return len(buff)
 
    def unlink(self, path):
        self.dbg("%-9s %s" % ("UNLINK:", path))
        self.rmn(path, False)
 
 
if __name__ == '__main__':
    if len(argv) != 2:
        print 'usage: %s <mountpoint>' % argv[0]
        exit(1)
    fuse = FUSE(MemFiS(), argv[1], foreground=False)