danboorufs: danboorufs.py annotate

author	Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
date	Wed, 04 Jul 2012 13:22:01 +0200
parents
children	63ccd8b0d615

rev	line source
0 215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	1 #!/usr/bin/env python2
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	2 # -- encoding: utf-8 --
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	3
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	4 from __future__ import with_statement
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	5
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	6 from errno import ENOENT, ENOTDIR
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	7 from os.path import realpath
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	8 from sys import argv, exit
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	9 from threading import Lock
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	10 from time import time
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	11
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	12 import os
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	13
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	14 from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	15
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	16
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	17 class Danbooru(LoggingMixIn, Operations):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	18 '''
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	19 Represent a list of images as a filesystem tree, with nice tag filtering.
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	20 '''
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	21
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	22 def __init__(self, tagfiles, root):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	23 '''
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	24 Takes a list of files containing the tags. They have to be named as the
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	25 image, with ".tags" at the end.
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	26 '''
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	27 self.paths = {}
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	28 self.files = {}
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	29 self.tags = {}
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	30 self.cache = {}
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	31
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	32 start = time()
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	33
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	34 for name in tagfiles:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	35 filename = name.replace('.tags', '')
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	36 basename = os.path.basename(filename)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	37 self.paths[basename] = filename
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	38 tags = []
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	39 self.files[basename] = tags
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	40 with open(name, 'r') as file:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	41 for line in file:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	42 for tag in line.split():
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	43 tag = tag.decode('UTF-8')
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	44 if '/' in tag:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	45 tag = tag.replace('/', u'�') #XXX
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	46 tags.append(tag)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	47 self.tags.setdefault(tag, []).append(basename)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	48
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	49 print('[%d] Index done.' % (time() - start))
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	50
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	51 self.root = root
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	52 self.rwlock = Lock()
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	53
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	54 def _split_path(self, path):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	55 if path == '/':
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	56 return (None, None)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	57
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	58 path = path[1:].split('/')
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	59 if filter(lambda tag: tag not in self.tags, path[:-1]):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	60 raise FuseOSError(ENOENT)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	61
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	62 if path[-1] in self.tags:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	63 return (path, None)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	64
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	65 if path[-1] not in self.paths:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	66 raise FuseOSError(ENOENT)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	67
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	68 return (path[:-1], self.paths[path[-1]])
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	69
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	70 def access(self, path, mode):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	71 self._split_path(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	72
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	73 def getattr(self, path, fh=None):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	74 tags, file = self._split_path(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	75 path = file if file else self.root
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	76 st = os.lstat(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	77 return dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime',
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	78 'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	79
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	80 getxattr = None
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	81 listxattr = None
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	82 def open(self, path, flags):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	83 tags, file = self._split_path(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	84 return os.open(file, flags)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	85
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	86 def read(self, path, size, offset, fh):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	87 with self.rwlock:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	88 os.lseek(fh, offset, 0)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	89 return os.read(fh, size)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	90
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	91 def readdir(self, path, fh):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	92 if path == '/':
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	93 return ['.', '..'] + self.tags.keys() + self.files.keys()
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	94
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	95 tags, file = self._split_path(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	96 if file:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	97 return FuseOSError(ENOTDIR)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	98
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	99 tags = set(tags)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	100
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	101 l = ' '.join(tags)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	102 if l in self.cache:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	103 return ['.', '..'] + self.cache[l]
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	104
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	105 # Get the list of the files corresponding to those tags.
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	106 files = reduce((lambda s, t: s.intersection(self.tags[t])), tags, set(self.files))
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	107
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	108 # Get the tags of those files.
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	109 taglist = reduce((lambda s, f: s.union(self.files[f])), files, set())
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	110 taglist -= tags
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	111
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	112 # Remove the tags that can’t precise the file list anymore.
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	113 rm = reduce((lambda s, f: s.intersection(self.files[f])), files, taglist)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	114 taglist -= rm
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	115
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	116 self.cache[l] = list(taglist) + list(files)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	117 return ['.', '..'] + self.cache[l]
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	118
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	119 readlink = os.readlink
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	120
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	121 def release(self, path, fh):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	122 return os.close(fh)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	123
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	124 def statfs(self, path):
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	125 tags, file = self._split_path(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	126 path = file if file else self.root
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	127 stv = os.statvfs(path)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	128 return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	129 'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	130 'f_frsize', 'f_namemax'))
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	131
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	132 utimens = os.utime
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	133
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	134
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	135 if __name__ == '__main__':
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	136 if len(argv) < 3:
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	137 print('usage: %s <tag file> [<tag file>...] <mountpoint>' % argv[0])
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	138 exit(1)
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	139
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	140 mountpoint = argv.pop()
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	141
215d51f2a82f Initial commit. Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> parents: diff changeset	142 fuse = FUSE(Danbooru(argv[1:], os.path.dirname(mountpoint)), mountpoint, foreground=True)

0

215d51f2a82f Initial commit.