annotate feed-push @ 32:5348758c622d draft

don't keep file descriptors open Signed-off-by: Changaco <changaco ατ changaco δοτ net>
author Changaco <changaco ατ changaco δοτ net>
date Sun, 05 Aug 2012 14:47:19 +0200
parents f5c5e6f4edc8
children adde3e451237
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
1 #!/usr/bin/env python2
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
3
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
4 # This program is free software: you can redistribute it and/or modify
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
5 # it under the terms of the GNU General Public License as published by
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
6 # the Free Software Foundation, either version 3 of the License, or
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
7 # (at your option) any later version.
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
8 #
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
9 # This program is distributed in the hope that it will be useful,
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
12 # GNU General Public License for more details.
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
13 #
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
14 # You should have received a copy of the GNU General Public License
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
16
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
17 import argparse
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
18 import calendar
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
19 from functools import partial, reduce
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
20 from glob import glob
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
21 import json
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
22 import os
7
09afbcd3efcb fix previous commit
Changaco <changaco ατ changaco δοτ net>
parents: 6
diff changeset
23 from os.path import abspath, dirname, isdir
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
24 import shlex
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
25 from subprocess import Popen, PIPE, STDOUT
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
26 import sys
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
27 from syslog import *
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
28 import time
21
3ecfeaba3c92 fix exception handling
Changaco <changaco ατ changaco δοτ net>
parents: 20
diff changeset
29 import traceback
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
30
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
31 import feedparser
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
32
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
33
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
34 # Constants
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
35
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
36 log_levels = ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERR', 'CRIT', 'ALERT', 'EMERG']
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
37
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
38 gamin_events = {
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
39 1:'GAMChanged', 2:'GAMDeleted', 3:'GAMStartExecuting', 4:'GAMStopExecuting',
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
40 5:'GAMCreated', 6:'GAMMoved', 7:'GAMAcknowledge', 8:'GAMExists', 9:'GAMEndExist'
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
41 }
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
42
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
43
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
44 # Generic utils
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
45
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
46 concat = lambda l: reduce(list.__add__, l, [])
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
47
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
48 def dict_append(d, k, v):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
49 d.setdefault(k, [])
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
50 d[k].append(v)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
51
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
52
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
53 # Logging
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
54
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
55 def log(*args):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
56 if len(args) == 1:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
57 priority, msg = LOG_INFO, args[0]
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
58 elif len(args) == 2:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
59 priority, msg = args
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
60 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
61 return
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
62 priority = 7 - priority
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
63 if priority < global_args.log_level:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
64 return
15
68a9b24a182a encode log messages in UTF8 before passing them to syslog
Changaco <changaco ατ changaco δοτ net>
parents: 14
diff changeset
65 if isinstance(msg, unicode):
68a9b24a182a encode log messages in UTF8 before passing them to syslog
Changaco <changaco ατ changaco δοτ net>
parents: 14
diff changeset
66 msg = msg.encode('utf8')
18
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
67 try:
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
68 if global_args.fork:
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
69 syslog(priority, msg)
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
70 else:
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
71 sys.stderr.write(log_levels[priority]+': '+msg+'\n')
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
72 except:
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
73 exit(1)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
74
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
75 def ignore_event(path, event):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
76 log(LOG_DEBUG, 'ignoring event '+gamin_events.get(event, str(event))+' on '+path)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
77
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
78
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
79 # Config parsing
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
80
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
81 def parse_config_file(config_path):
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
82 try:
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
83 config_fd = open(config_path)
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
84 except IOError as e:
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
85 return log('failed to open "'+line+'": '+str(e))
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
86 feeds_paths = config_to_feed_paths_to_commands[config_path] = {}
11
0b28010f6c78 fix config parsing
Changaco <changaco ατ changaco δοτ net>
parents: 10
diff changeset
87 cmd = []
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
88 log('parsing config file '+config_path)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
89 config_fd.seek(0)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
90 for i, line in enumerate(config_fd):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
91 line = line.strip()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
92 if len(line) == 0 or line[0] == '#':
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
93 continue
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
94 if line[0] == '%':
11
0b28010f6c78 fix config parsing
Changaco <changaco ατ changaco δοτ net>
parents: 10
diff changeset
95 if isinstance(cmd, str):
0b28010f6c78 fix config parsing
Changaco <changaco ατ changaco δοτ net>
parents: 10
diff changeset
96 cmd = []
0b28010f6c78 fix config parsing
Changaco <changaco ατ changaco δοτ net>
parents: 10
diff changeset
97 cmd.append(line[1:].rstrip(';'))
0b28010f6c78 fix config parsing
Changaco <changaco ατ changaco δοτ net>
parents: 10
diff changeset
98 elif not cmd:
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
99 log(LOG_ERR, 'missing command in file '+config_path+' before line '+str(i))
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
100 return
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
101 else:
11
0b28010f6c78 fix config parsing
Changaco <changaco ατ changaco δοτ net>
parents: 10
diff changeset
102 cmd = '; '.join(cmd)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
103 for feed_path in glob(line):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
104 feed_path = abspath(feed_path)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
105 dict_append(feeds_paths, feed_path, cmd)
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
106 if not feed_path in watched_feeds:
18
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
107 monitor.watch_file(feed_path, handler(handle_feed_change))
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
108 watched_feeds.add(feed_path)
12
ddbbd4ee5d47 log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents: 11
diff changeset
109 log('now watching '+feed_path)
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
110 config_fd.close()
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
111
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
112
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
113 # Gamin callbacks
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
114
18
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
115 def handler(f):
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
116 def g(path, event):
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
117 try:
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
118 f(path, event)
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
119 except:
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
120 log(LOG_CRIT, traceback.format_exc())
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
121 return g
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
122
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
123 def handle_config_change(path, event):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
124 path = abspath(path)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
125 if os.path.isdir(path):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
126 ignore_event(path, event)
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
127 elif not path in config_to_feed_paths_to_commands:
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
128 open_config(path, event)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
129 elif event in [gamin.GAMChanged, gamin.GAMDeleted]:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
130 update_config(path, event)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
131 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
132 ignore_event(path, event)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
133
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
134 def open_config(path, event):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
135 if event in [gamin.GAMCreated, gamin.GAMExists]:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
136 if (not path.endswith('.conf') or path[0] == '.') and not hasattr(global_args.config, 'read'):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
137 return log('ignoring '+path+' (not a valid config file name)')
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
138 parse_config_file(path)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
139 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
140 ignore_event(path, event)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
141
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
142 def update_config(path, event):
10
89f581ebf4b2 fix update_config function
Changaco <changaco ατ changaco δοτ net>
parents: 7
diff changeset
143 feeds_paths = set(concat(d.keys() for d in config_to_feed_paths_to_commands.values()))
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
144 if event == gamin.GAMChanged:
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
145 parse_config_file(path)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
146 elif event == gamin.GAMDeleted:
25
0946b4e64b20 fix a NameError
Changaco <changaco ατ changaco δοτ net>
parents: 24
diff changeset
147 log('removing actions from deleted config file '+path)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
148 config_to_feed_paths_to_commands.pop(path)
10
89f581ebf4b2 fix update_config function
Changaco <changaco ατ changaco δοτ net>
parents: 7
diff changeset
149 new_feeds_paths = set(concat(d.keys() for d in config_to_feed_paths_to_commands.values()))
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
150 for feed_path in feeds_paths.difference(new_feeds_paths):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
151 monitor.stop_watch(feed_path)
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
152 watched_feeds.discard(feed_path)
12
ddbbd4ee5d47 log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents: 11
diff changeset
153 log('stopped watching '+feed_path)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
154
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
155 def handle_feed_change(path, event):
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
156 if event in [gamin.GAMCreated, gamin.GAMExists, gamin.GAMChanged]:
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
157 try:
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
158 feed_fd = open(path)
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
159 except IOError as e:
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
160 return log('failed to open "'+path+'": '+str(e))
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
161 feed = feedparser.parse(feed_fd.read())
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
162 feed_fd.close()
23
5bb7c2939da0 add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents: 22
diff changeset
163 i = 0
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
164 for entry in reversed(feed.entries):
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
165 if entry.id in state['id_cache'].get(path, []) or \
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
166 not global_args.flood and calendar.timegm(entry.published_parsed) < time.time() - 86400:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
167 continue
23
5bb7c2939da0 add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents: 22
diff changeset
168 i += 1
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
169 for feed_path_to_commands in config_to_feed_paths_to_commands.values():
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
170 for cmd in feed_path_to_commands.get(path, []):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
171 run_command(format_cmd(cmd, feed=feed.feed, entry=entry), entry.content[0].value)
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
172 state['id_cache'][path] = [entry.id for entry in feed.entries]
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
173 save_state()
23
5bb7c2939da0 add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents: 22
diff changeset
174 if i == 0:
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
175 log('no new entry in %s' % path)
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
176 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
177 ignore_event(path, event)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
178
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
179 def save_state():
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
180 global_args.state_file.truncate(0)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
181 json.dump(state, global_args.state_file)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
182 global_args.state_file.flush()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
183
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
184
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
185 # Commands utils
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
186
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
187 def format_cmd(cmd, **kwargs):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
188 """The safe equivalent of str.format() for shell commands, meaning interpolated variables can't do shell injections (I hope)."""
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
189 r = u''
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
190 for arg in shlex.split(cmd.encode('utf8')):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
191 a = arg.decode('utf8')
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
192 b = a.format(**kwargs)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
193 if a != b:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
194 r += u" '" + b.replace(u"'", u'\'"\'"\'') + u"'"
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
195 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
196 r += u' ' + arg
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
197 return r.lstrip()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
198
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
199 def run_command(cmd, input):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
200 p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
201 output = p.communicate(input.encode('utf8'))[0].decode('utf8')
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
202 if p.returncode != 0:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
203 log(LOG_ERR, 'command failed: '+cmd+'\n'+output)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
204 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
205 log(LOG_INFO, 'successfully executed '+cmd)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
206 log(LOG_DEBUG, '===== output:\n'+output)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
207
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
208
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
209 # Argparse utils
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
210
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
211 def AbsPath(next_type=None):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
212 def f(s):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
213 p = abspath(s)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
214 if next_type is not None:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
215 return next_type(p)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
216 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
217 return p
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
218 return f
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
219
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
220 class Apply(argparse.Action):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
221 def __init__(self, f, *args, **kwargs):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
222 super(self.__class__, self).__init__(**kwargs)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
223 self.f = f
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
224 def __call__(self, parser, namespace, values, option_string=None):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
225 setattr(namespace, self.dest, self.f(values[0]))
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
226
6
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
227 def MakeDirs(next_type=None):
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
228 def f(s):
7
09afbcd3efcb fix previous commit
Changaco <changaco ατ changaco δοτ net>
parents: 6
diff changeset
229 d = dirname(s)
14
5c26fc1adbac minor fix in automatic directory creation
Changaco <changaco ατ changaco δοτ net>
parents: 12
diff changeset
230 if d and not isdir(d):
6
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
231 try:
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
232 os.makedirs(d)
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
233 except OSError as e:
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
234 raise argparse.ArgumentTypeError(str(e))
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
235 if next_type is not None:
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
236 return next_type(s)
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
237 else:
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
238 return s
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
239 return f
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
240
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
241 def Directory(s):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
242 try:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
243 os.listdir(s)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
244 return s
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
245 except OSError as e:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
246 raise argparse.ArgumentTypeError(str(e))
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
247
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
248 def File(flags):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
249 def f(s):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
250 try:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
251 return os.fdopen(os.open(s, flags), 'w')
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
252 except OSError as e:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
253 raise argparse.ArgumentTypeError(str(e))
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
254 return f
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
255
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
256 class First(argparse.Action):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
257 def __call__(self, parser, namespace, values, option_string=None):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
258 setattr(namespace, self.dest, values[0])
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
259
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
260 def FirstOf(*types, **kwargs):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
261 kwargs.setdefault('error', 'argument "{}" is not valid')
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
262 def f(s):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
263 for t in types:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
264 try:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
265 return t(s)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
266 except:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
267 pass
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
268 raise argparse.ArgumentTypeError(error.format(s))
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
269 return f
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
270
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
271
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
272 # Main
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
273
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
274 if __name__ == '__main__':
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
275
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
276 p = argparse.ArgumentParser()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
277 p.add_argument('config', type=FirstOf(AbsPath(argparse.FileType('r')), AbsPath(Directory), error='"{}" is neither a file nor a directory'), help='either a file or a directory')
6
455cd8c78862 create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents: 1
diff changeset
278 p.add_argument('state_file', type=MakeDirs(argparse.FileType('a+')), help='e.g. /var/lib/feed-push/state')
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
279 p.add_argument('--flood', default=False, action='store_true', help='push all articles on startup instead of ignoring the ones older than 24h (useful for debugging)')
16
183f4f544987 merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents: 15
diff changeset
280 p.add_argument('--fork', metavar='pid-file', nargs=1, type=File(os.O_WRONLY|os.O_CREAT|os.O_EXCL), action=First, help='daemonize and log to syslog')
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
281 p.add_argument('--log-level', nargs=1, default=1, choices=log_levels, action=partial(Apply, log_levels.index), help='default is INFO')
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
282 global_args = p.parse_args()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
283
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
284 if global_args.fork:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
285 pid = os.fork()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
286 if pid != 0:
16
183f4f544987 merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents: 15
diff changeset
287 exit(0)
19
f22369c00480 fix double forking
Changaco <changaco ατ changaco δοτ net>
parents: 18
diff changeset
288 os.setsid()
16
183f4f544987 merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents: 15
diff changeset
289 pid = os.fork()
183f4f544987 merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents: 15
diff changeset
290 if pid != 0:
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
291 global_args.fork.write(str(pid))
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
292 exit(0)
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
293 openlog(facility=LOG_DAEMON)
17
421a3416e768 close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents: 16
diff changeset
294 null = open('/dev/null', 'r+')
421a3416e768 close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents: 16
diff changeset
295 for f in [sys.stdin, sys.stdout, sys.stderr]:
421a3416e768 close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents: 16
diff changeset
296 f.flush()
421a3416e768 close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents: 16
diff changeset
297 os.dup2(null.fileno(), f.fileno())
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
298
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
299 state = {'id_cache': {}}
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
300 saved_state = global_args.state_file.read().strip()
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
301 if len(saved_state) > 0:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
302 state.update(json.loads(saved_state))
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
303 del saved_state
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
304
22
b763ca084088 import gamin in __main__
Changaco <changaco ατ changaco δοτ net>
parents: 21
diff changeset
305 import gamin
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
306 monitor = gamin.WatchMonitor()
32
5348758c622d don't keep file descriptors open
Changaco <changaco ατ changaco δοτ net>
parents: 31
diff changeset
307 watched_feeds = set()
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
308 config_to_feed_paths_to_commands = {}
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
309 if hasattr(global_args.config, 'read'):
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
310 os.chdir(os.path.dirname(global_args.config.name))
18
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
311 monitor.watch_file(global_args.config.name, handler(handle_config_change))
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
312 else:
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
313 os.chdir(global_args.config)
18
868a339b19f8 log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents: 17
diff changeset
314 monitor.watch_directory(global_args.config, handler(handle_config_change))
1
a68d7feeba88 initial commit
Changaco <changaco ατ changaco δοτ net>
parents:
diff changeset
315
20
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
316 if global_args.fork:
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
317 while True:
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
318 monitor.handle_one_event()
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
319 else:
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
320 try:
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
321 while True:
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
322 monitor.handle_events()
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
323 time.sleep(1)
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
324 except KeyboardInterrupt:
94f3b22beec9 use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents: 19
diff changeset
325 exit(0)