MemCache-Preload - Revision history

TitusX: /* Mount, Preload, Unmount */

2013-04-27T09:47:55Z

‎Mount, Preload, Unmount

TitusX: /* Ingredients */

2013-04-27T09:46:31Z

‎Ingredients

TitusX: /* MemFiS Filesystem */

2012-04-18T21:42:27Z

‎MemFiS Filesystem

TitusX: /* Server Setup */

2012-04-18T21:39:30Z

‎Server Setup

TitusX: /* MemFiS Filesystem */

2012-04-18T21:34:08Z

‎MemFiS Filesystem

TitusX: /* Ingredients */

2012-04-18T21:33:29Z

‎Ingredients

TitusX: /* MemFiS Filesystem */

2012-04-18T21:32:17Z

‎MemFiS Filesystem

TitusX: /* MemFiS Filesystem */

2012-04-18T21:30:31Z

‎MemFiS Filesystem

Show changes

TitusX: /* Ingredients */

2012-04-18T21:28:45Z

‎Ingredients

TitusX: /* Ingredients */

2011-05-02T13:48:26Z

‎Ingredients

New page

= Ingredients =

'''This setups is only a proof of concept = means: experimental, but works for me.'''

# serve the equests
#* '''[http://wiki.nginx.org/HttpMemcachedModule nginx]''': webserver with memcache api
#* '''[http://www.couchbase.com/downloads membase]''': persistent memcache server
# preload memcache
#* python: a programming language
#* fuse: file system in user space
#* '''[http://code.google.com/p/fusepy/source/browse/trunk/fuse.py fuse.py]''': python library for fuse
#* '''[http://bazaar.launchpad.net/~python-memcached-team/python-memcached/trunk/view/head:/memcache.py memcache.py]''': python library for memcache
#* '''memfis.py''': experimental fuse file system

= Server Setup =

== Installation ==
<geshi lang=bash>
apt-get install nginx-full
cp /etc/nginx/sites-available/default /etc/nginx/sites-available/default.dpkg-dist
vi /etc/nginx/sites-available/default
dpkg -i membase-server-community_<arch>_<version>.deb
# Browser: http://<hostname>:8091
</geshi>

== /etc/nginx/sites-available/default ==
<geshi lang=nginx>
server {
listen 80;
server_name <webserver>;
root /var/www/;

location / {
index index.html;
default_type text/plain;
set $memcached_key memfis://<hostname>$uri;
memcached_pass 127.0.0.1:11211;
}
}
</geshi>
* to do: error, fallback ...

= Preload Memcache =

== Preparation ==
<geshi lang=bash>
mkdir /mnt/memfis
mkdir memfis.d
cd memfis.d
vi memfis.py fuse.py memcache.py
chmod +x memfis.py
</geshi>

== Mount, Preload, Unmount ==
<geshi lang=bash>
memfis.d/memfis.py /mnt/memfis
cp -a <source>/* /mnt/memfis
sudo unmount /mnt/memfis
</geshi>

== MemFiS Filesystem ==

<geshi lang=python>
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# (c) 2011, titusx at gmx.de

#import syslog, binascii
from time import time, strftime
from sys import argv, exit
from socket import gethostname
from os import getuid, getgid, O_RDONLY, O_WRONLY, O_RDWR
from errno import *
from stat import S_IFDIR, S_IFREG
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
from json import dumps, loads
import memcache as mc
#import pylibmc as mc

class Auxiliary(object):

def ini(self): # custom ini
self.debugp = False #'/var/log/memfis-debug.log'
self.server = ["127.0.0.1:11211"]
self.memfis = None
self.noatim = True
self.prefix = "memfis://" + gethostname()
self.countr = self.prefix + "/?cntr"
self.erased = self.prefix + "/?free"
self.initim = self.now()
self.iniuid = getuid()
self.inigid = getgid()

def now(self): # now
return time()

def mka(self): # mkattr
Attr = dict(
st_ino = 0,
#st_dev = 0,
#st_rdev = 0,
#st_blksize = 20000000,
#st_blocks = 1,
st_nlink = 2,
st_size = 4,
st_mode = 0,
st_uid = self.iniuid,
st_gid = self.inigid,
st_atime = self.initim,
st_mtime = self.initim,
st_ctime = self.initim )
return Attr

def dbg(self, mssg): # debug
if self.debugp:
stamp = strftime("%Y%m%d-%H:%M.%S")
log = open(self.debugp, 'a')
log.write("[%s] %s\n" % (stamp, mssg))
log.close()
return

def p2a(self, path): # attr
return self.prefix + path + "?attr"

def p2x(self, path): # xattr
return self.prefix + path + "?xttr"

def p2d(self, path): # data
return self.prefix + path

def c2n(self, cntr): # counter to node
return self.prefix + "/?node=%08i" % (cntr)

def par(self, path): # parent+file
if path == "/":
return None
Splt = path.rsplit("/",1)
if Splt[0] == "":
Splt[0] = "/"
return Splt

def cnt(self, path): # counter
Cntr = self.memfis.incr(self.countr)
try:
self.memfis.add(self.c2n(Cntr), path)
except:
raise FuseOSError(EEXIST)
return Cntr

def fre(self, cntr): # erased
self.memfis.incr(self.erased)
return self.c2n(cntr)

def enc(self, pobj):
return dumps(pobj, sort_keys=True, indent=1)

def dec(self, strg):
return loads(strg)

class MemFiS(Operations, Auxiliary):

def __init__(self, *args, **kw):
self.ini()
self.dbg("__INIT__")
self.memfis = mc.Client(self.server)
if self.debugp:
self.memfis.flush_all()
Cntr = self.memfis.get(self.countr)
if Cntr == None:
self.memfis.set(self.countr, 0)
self.memfis.set(self.erased, 0)
Attr = self.memfis.get(self.p2a("/"))
if Attr == None:
self.mkdir("/", 493)

def __del__(self):
self.dbg("__DEL__")
if self.debugp:
self.memfis.flush_all()
self.memfis.disconnect_all()

def chmod(self, path, mode):
self.dbg("CHOWN: %s (mode: '%s'=%i)" % (path, oct(mode), mode))
JAttr = self.memfis.get(self.p2a(path))
if JAttr == None:
raise FuseOSError(ENOENT) # daten nicht gefunden
else:
Attr = self.dec(JAttr)
if Attr['st_mode'] == mode:
return 0
Attr['st_mode'] = mode
Attr['st_mtime'] = self.now()
self.memfis.set(self.p2a(path), self.enc(Attr))

def chown(self, path, usid, grid):
self.dbg("CHMOD: %s (uid: %i, gid: %i)" % (path, usid, grid))
JAttr = self.memfis.get(self.p2a(path))
if JAttr == None:
raise FuseOSError(ENOENT) # daten nicht gefunden
else:
Attr = self.dec(JAttr)
Dirt = False
if Attr['st_uid'] != usid and usid >= 0:
Attr['st_uid'] = usid
Dirt = True
if Attr['st_gid'] != grid and grid >= 0:
Attr['st_gid'] = grid
Dirt = True
if Dirt == False:
return 0
Attr['st_mtime'] = self.now()
self.memfis.set(self.p2a(path), self.enc(Attr))

def addnode(self, path, mode, dire):
self.dbg("addnode: %s (mode: '%s'=%i)" % (path, oct(mode), mode))
JAttr = self.memfis.get(self.p2a(path))
if JAttr != None:
raise FuseOSError(EEXIST) # datei existiert
Splt = self.par(path)
if Splt != None:
Attr = self.dec(self.memfis.get(self.p2a(Splt[0])))
List = self.dec(self.memfis.get(self.p2d(Splt[0])))
List.append(Splt[1])
List.sort()
Attr['st_size'] = len(self.enc(List))
Attr['st_mtime'] = self.now()
Attr['st_atime'] = Attr['st_mtime']
if dire == True:
Attr['st_nlink'] += 1
self.memfis.set(self.p2a(Splt[0]), self.enc(Attr))
self.memfis.set(self.p2d(Splt[0]), self.enc(List))
Attr = self.mka()
Attr['st_ino'] = self.cnt(path)
Attr['st_uid'] = self.iniuid
Attr['st_gid'] = self.inigid
Attr['st_ctime'] = self.now()
Attr['st_mtime'] = Attr['st_ctime']
Attr['st_atime'] = Attr['st_ctime']
if dire == True:
Attr['st_nlink'] = 2
Attr['st_mode'] = S_IFDIR | mode
Attr['st_size'] = len(self.enc(['.', '..']))
else:
Attr['st_nlink'] = 1
Attr['st_mode'] = S_IFREG | mode
Attr['st_size'] = 0
self.memfis.set(self.p2a(path), self.enc(Attr))
return Attr['st_ino']

def create(self, path, mode, fi=None):
self.dbg("CREATE: %s (mode: '%s'=%i)" % (path, oct(mode), mode))
return self.addnode(path, mode, False)

def getattr(self, path, fhdl=None):
self.dbg("GETATTR: " + path)
JAttr = self.memfis.get(self.p2a(path))
if JAttr == None:
raise FuseOSError(ENOENT) # daten nicht gefunden
else:
return self.dec(JAttr)

def mkdir(self, path, mode):
self.dbg("MKDIR: %s (mode: '%s'=%i)" % (path, oct(mode), mode))
self.addnode(path, mode, True)
self.memfis.set(self.p2d(path), self.enc(['.', '..']))

def read(self, path, size, oset, fhdl):
self.dbg("READ: %s (size: %i, offset: %i)" % (path, size, oset))
if oset != 0:
raise FuseOSError(ENOSYS) # nicht implementiert
JAttr = self.memfis.get(self.p2a(path))
if JAttr == None:
raise FuseOSError(ENOENT) # daten nicht gefunden
else:
Attr = self.dec(JAttr)
if self.noatim == False:
Attr['st_atime'] = now()
self.memfis.set(self.p2a(path), self.enc(Attr))
Data = self.memfis.get(self.p2d(path))
return Data

def readdir(self, path, fhdl):
self.dbg("READDIR: " + path)
List = []
for e in self.dec(self.read(path, 0, 0, fhdl)):
List.append( e.encode('ascii') )
return List

def delnode(self, path, dire):
self.dbg("delnode: " + path)
if path == "/":
raise FuseOSError(ENOPERM)
JAttr = self.memfis.get(self.p2a(path))
if JAttr == None:
raise FuseOSError(ENOENT) # daten nicht gefunden
else:
Attr = self.dec(JAttr)
if dire == True and Attr['st_nlink'] > 2:
raise FuseOSError(ENOTEMPTY) # verzeichnis nicht leer
if dire == False and Attr['st_nlink'] != 1:
raise FuseOSError(ENOSYS) # nicht implementiert
self.memfis.delete_multi([self.p2a(path), self.p2d(path), self.fre(Attr['st_ino'])])
Splt = self.par(path)
if Splt != None:
Attr = self.dec(self.memfis.get(self.p2a(Splt[0])))
List = self.dec(self.memfis.get(self.p2d(Splt[0])))
List.remove(Splt[1])
Attr['st_size'] = len(self.enc(List))
Attr['st_mtime'] = self.now()
Attr['st_atime'] = Attr['st_mtime']
if dire == True:
Attr['st_nlink'] -= 1
self.memfis.set(self.p2a(Splt[0]), self.enc(Attr))
self.memfis.set(self.p2d(Splt[0]), self.enc(List))

def rmdir(self, path):
self.dbg("RMDIR: " + path)
self.delnode(path, True)

def write(self, path, buff, oset, fdhl=None):
self.dbg("WRITE: %s (buffer: %i, offset: %i)" % (path, len(buff), oset))
JAttr = self.memfis.get(self.p2a(path))
if JAttr == None:
raise FuseOSError(ENOENT) # daten nicht gefunden
else:
Attr = self.dec(JAttr)
if Attr['st_size'] + len(buff) > 20000000:
raise FuseOSError(EFBIG) # datei zu gross
Attr['st_size'] += len(buff)
Attr['st_mtime'] = self.now()
Attr['st_atime'] = Attr['st_mtime']
self.memfis.replace(self.p2a(path), self.enc(Attr))
if oset == 0:
self.memfis.add(self.p2d(path), buff)
else:
self.memfis.append(self.p2d(path), buff)
return len(buff)

def unlink(self, path):
self.dbg("UNLINK: " + path)
self.delnode(path, False)

if __name__ == '__main__':
if len(argv) != 2:
print 'usage: %s <mountpoint>' % argv[0]
exit(1)
fuse = FUSE(MemFiS(), argv[1], foreground=False)
</geshi>

@@ Line 266: / Line 266: @@
          # ONLY FOR TESTING - REMOVE FOR PRODUCTION
          # deletes the whole database at startup
-         if self.debugp:
+         #if self.debugp:
-            self.mcache.flush_all()
+        #    self.mcache.flush_all()
          # start a new file system if nothin exitst
          Cntr = self.mcache.get(self.countr)
@@ Line 282: / Line 282: @@
          # ONLY FOR TESTING - REMOVE FOR PRODUCTION
          # delete the whole database at shutdown
-         if self.debugp:
+         #if self.debugp:
-            self.mcache.flush_all()
+        #    self.mcache.flush_all()
          # disconnect from the server
          self.mcache.disconnect_all()
@@ Line 327: / Line 327: @@
              self.mcache.set(self.p2a(path), self.enc(Attr))
-     # ...
+     # creates an inode with file attributes
      def create(self, path, mode):
          self.dbg("%-9s %s (mode: '%s'=%i)" % ("CREATE:", path, oct(mode), mode))
@@ Line 347: / Line 347: @@
          self.mkn(path, S_IFDIR|mode, len(Data))
          self.mcache.set(self.p2d(path), Data)
      def read(self, path, size, oset, fhdl):
@@ Line 383: / Line 375: @@
          self.dbg("%-9s %s" % ("READLINK:", path))
          return self.mcache.get(self.p2d(path))
      def rmdir(self, path):
          self.dbg("%-9s %s" % ("RMDIR:", path))
          self.rmn(path, True)
      def symlink(self, path, orig):

@@ Line 64: / Line 64: @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-"""memcache filesystem"""
+"""memcache filesystem, v.20120415"""
 """(c) 2011-2012, titusx at gmx.de"""

@@ Line 79: / Line 79: @@
 class Auxiliary(object):
@@ Line 249: / Line 250: @@
-# our file system class
+# the fuse file system class
 class MemFiS(Operations, Auxiliary):

@@ Line 56: / Line 56: @@
 == Mount, Preload, Unmount ==
 <geshi lang=bash>
-memfis.d/memfis.py /mnt/memfis
+./memfis.py /mnt/memfis
 cp -a <source>/* /mnt/memfis
 sudo unmount /mnt/memfis

@@ Line 3: / Line 3: @@
 '''This setups is only a proof of concept = means: experimental, but works for me.'''
-# serve the equests
+# serve the requests
 #* '''[http://wiki.nginx.org/HttpMemcachedModule nginx]''': webserver with memcache api
 #* '''[http://www.couchbase.com/downloads couchbase]''': persistent memcache server

@@ Line 39: / Line 39: @@
+}
 </geshi>
-* to do: error, fallback ...
+To do:
 = Preload Memcache =

@@ Line 11: / Line 11: @@
 #* '''[http://code.google.com/p/fusepy/source/browse/trunk/fuse.py fuse.py]''': python library for fuse
 #* '''[http://bazaar.launchpad.net/~python-memcached-team/python-memcached/trunk/view/head:/memcache.py memcache.py]''': python library for memcache
-#* '''memfis.py''': experimental fuse file system
+#* '''memfis.py''': experimental fuse file system (new: symbolic links)
 = Server Setup =

@@ Line 5: / Line 5: @@
 # serve the equests
 #* '''[http://wiki.nginx.org/HttpMemcachedModule nginx]''': webserver with memcache api
-#* '''[http://www.couchbase.com/downloads membase]''': persistent memcache server
+#* '''[http://www.couchbase.com/downloads couchbase]''': persistent memcache server
 # preload memcache
 #* python: a programming language