view feed-push @ 3:0d4378996672 draft

improve setup script Signed-off-by: Changaco <changaco ατ changaco δοτ net>
author Changaco <changaco ατ changaco δοτ net>
date Mon, 16 Apr 2012 00:24:59 +0200
parents a68d7feeba88
children 455cd8c78862
line wrap: on
line source

#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import calendar
from   functools  import partial, reduce
from   glob       import glob
import json
import os
from   os.path    import abspath
import shlex
from   subprocess import Popen, PIPE, STDOUT
import sys
from   syslog     import *
import time

import feedparser
import gamin


# Constants

log_levels = ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERR', 'CRIT', 'ALERT', 'EMERG']

gamin_events = {
    1:'GAMChanged', 2:'GAMDeleted', 3:'GAMStartExecuting', 4:'GAMStopExecuting',
    5:'GAMCreated', 6:'GAMMoved', 7:'GAMAcknowledge', 8:'GAMExists', 9:'GAMEndExist'
}


# Generic utils

concat = lambda l: reduce(list.__add__, l, [])

def dict_append(d, k, v):
    d.setdefault(k, [])
    d[k].append(v)


# Logging

def log(*args):
    if len(args) == 1:
        priority, msg = LOG_INFO, args[0]
    elif len(args) == 2:
        priority, msg = args
    else:
        return
    priority = 7 - priority
    if priority < global_args.log_level:
        return
    if global_args.syslog:
        syslog(*args)
    else:
        sys.stderr.write(log_levels[priority]+': '+msg+'\n')

def ignore_event(path, event):
    log(LOG_DEBUG, 'ignoring event '+gamin_events.get(event, str(event))+' on '+path)


# Config parsing

def parse_config_file(config_fd):
    feeds_paths = config_to_feed_paths_to_commands[config_fd.name] = {}
    cmd = None
    next_cmd = []
    config_fd.seek(0)
    for i, line in enumerate(config_fd):
        line = line.strip()
        if len(line) == 0 or line[0] == '#':
            continue
        if line[0] == '%':
            next_cmd.append(line[1:].rstrip(';'))
        elif not next_cmd:
            log(LOG_ERR, 'missing command in file '+config_fd.name+' before line '+str(i))
            return
        else:
            cmd = '; '.join(next_cmd)
            next_cmd = []
            for feed_path in glob(line):
                feed_path = abspath(feed_path)
                dict_append(feeds_paths, feed_path, cmd)
                if not feed_path in path_to_feed_fd:
                    monitor.watch_file(feed_path, handle_feed_change)
    log('successfully parsed config file '+config_fd.name)


# Gamin callbacks

def handle_config_change(path, event):
    path = abspath(path)
    if os.path.isdir(path):
        ignore_event(path, event)
    elif not path in path_to_config_fd:
        open_config(path, event)
    elif event in [gamin.GAMChanged, gamin.GAMDeleted]:
        update_config(path, event)
    else:
        ignore_event(path, event)

def open_config(path, event):
    if event in [gamin.GAMCreated, gamin.GAMExists]:
        if (not path.endswith('.conf') or path[0] == '.') and not hasattr(global_args.config, 'read'):
            return log('ignoring '+path+' (not a valid config file name)')
        try:
            config_fd = open(path)
        except IOError as e:
            return log('failed to open "'+line+'" '+str(e))
        path_to_config_fd[path] = config_fd
        parse_config_file(config_fd)
    else:
        ignore_event(path, event)

def update_config(path, event):
    feeds_paths = set(concat(config_to_feed_paths_to_commands.values()))
    if event == gamin.GAMChanged:
        log('updating actions from modified config file '+config_fd.name)
        parse_config_file(path_to_config_fd[path])
    elif event == gamin.GAMDeleted:
        log('removing actions from deleted config file '+config_fd.name)
        config_to_feed_paths_to_commands.pop(path)
        path_to_config_fd.pop(path).close()
    new_feeds_paths = set(concat(config_to_feed_paths_to_commands.values()))
    for feed_path in feeds_paths.difference(new_feeds_paths):
        monitor.stop_watch(feed_path)
        if feed_path in path_to_feed_fd:
            path_to_feed_fd.pop(feed_path).close()

def handle_feed_change(path, event):
    if path not in path_to_feed_fd:
        if event in [gamin.GAMCreated, gamin.GAMExists, gamin.GAMChanged]:
            try:
                feed_fd = path_to_feed_fd[path] = open(path)
            except IOError as e:
                return log('failed to open "'+path+'": '+str(e))
            feed = feedparser.parse(feed_fd.read())
            handle_feed_change(path, gamin.GAMChanged)
        else:
            ignore_event(path, event)
    elif event == gamin.GAMChanged:
        feed_fd = path_to_feed_fd[path]
        feed_fd.seek(0)
        feed = feedparser.parse(feed_fd.read())
        for entry in reversed(feed.entries):
            if entry.id in state['id_cache'].get(feed_fd.name, []) or \
               not global_args.flood and calendar.timegm(entry.published_parsed) < time.time() - 86400:
                continue
            for feed_path_to_commands in config_to_feed_paths_to_commands.values():
                for cmd in feed_path_to_commands.get(path, []):
                    run_command(format_cmd(cmd, feed=feed.feed, entry=entry), entry.content[0].value)
        state['id_cache'][feed_fd.name] = [entry.id for entry in feed.entries]
        save_state()
    elif event == gamin.GAMDeleted:
        path_to_feed_fd.pop(path).close()
    else:
        ignore_event(path, event)

def save_state():
    global_args.state_file.truncate(0)
    json.dump(state, global_args.state_file)
    global_args.state_file.flush()


# Commands utils

def format_cmd(cmd, **kwargs):
    """The safe equivalent of str.format() for shell commands, meaning interpolated variables can't do shell injections (I hope)."""
    r = u''
    for arg in shlex.split(cmd.encode('utf8')):
        a = arg.decode('utf8')
        b = a.format(**kwargs)
        if a != b:
            r += u" '" + b.replace(u"'", u'\'"\'"\'') + u"'"
        else:
            r += u' ' + arg
    return r.lstrip()

def run_command(cmd, input):
    p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True)
    output = p.communicate(input.encode('utf8'))[0].decode('utf8')
    if p.returncode != 0:
        log(LOG_ERR, 'command failed: '+cmd+'\n'+output)
    else:
        log(LOG_INFO, 'successfully executed '+cmd)
        log(LOG_DEBUG, '===== output:\n'+output)


# Argparse utils

def AbsPath(next_type=None):
    def f(s):
        p = abspath(s)
        if next_type is not None:
            return next_type(p)
        else:
            return p
    return f

class Apply(argparse.Action):
    def __init__(self, f, *args, **kwargs):
        super(self.__class__, self).__init__(**kwargs)
        self.f = f
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, self.f(values[0]))

def Directory(s):
    try:
        os.listdir(s)
        return s
    except OSError as e:
        raise argparse.ArgumentTypeError(str(e))

def File(flags):
    def f(s):
        try:
            return os.fdopen(os.open(s, flags), 'w')
        except OSError as e:
            raise argparse.ArgumentTypeError(str(e))
    return f

class First(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values[0])

def FirstOf(*types, **kwargs):
    kwargs.setdefault('error', 'argument "{}" is not valid')
    def f(s):
        for t in types:
            try:
                return t(s)
            except:
                pass
        raise argparse.ArgumentTypeError(error.format(s))
    return f


# Main

if __name__ == '__main__':

    p = argparse.ArgumentParser()
    p.add_argument('config', type=FirstOf(AbsPath(argparse.FileType('r')), AbsPath(Directory), error='"{}" is neither a file nor a directory'), help='either a file or a directory')
    p.add_argument('state_file', type=argparse.FileType('a+'), help='e.g. /var/lib/feed-push/state')
    p.add_argument('--flood', default=False, action='store_true', help='push all articles on startup instead of ignoring the ones older than 24h (useful for debugging)')
    p.add_argument('--fork', metavar='pid-file', nargs=1, type=File(os.O_WRONLY|os.O_CREAT|os.O_EXCL), action=First, help='useful in init scripts')
    p.add_argument('--log-level', nargs=1, default=1, choices=log_levels, action=partial(Apply, log_levels.index), help='default is INFO')
    p.add_argument('--syslog', default=False, action='store_true', help='log to syslog instead of stderr')
    global_args = p.parse_args()

    if global_args.fork:
        pid = os.fork()
        if pid != 0:
            global_args.fork.write(str(pid))
            exit(0)

    if global_args.syslog:
        openlog(facility=LOG_DAEMON)

    state = {'id_cache': {}}
    saved_state = global_args.state_file.read().strip()
    if len(saved_state) > 0:
        state.update(json.loads(saved_state))
    del saved_state

    monitor = gamin.WatchMonitor()
    path_to_feed_fd = {}
    path_to_config_fd = {}
    config_to_feed_paths_to_commands = {}
    if hasattr(global_args.config, 'read'):
        os.chdir(os.path.dirname(global_args.config.name))
        monitor.watch_file(global_args.config.name, handle_config_change)
    else:
        os.chdir(global_args.config)
        monitor.watch_directory(global_args.config, handle_config_change)

    while True:
        monitor.handle_one_event()