Mercurial > feed-push
annotate feed-push @ 23:5bb7c2939da0 draft
add some debug logging
Signed-off-by: Changaco <changaco ατ changaco δοτ net>
author | Changaco <changaco ατ changaco δοτ net> |
---|---|
date | Sat, 04 Aug 2012 16:57:38 +0200 |
parents | b763ca084088 |
children | 96e2d5ffcd23 |
rev | line source |
---|---|
1 | 1 #!/usr/bin/env python2 |
2 # -*- coding: utf-8 -*- | |
3 | |
4 # This program is free software: you can redistribute it and/or modify | |
5 # it under the terms of the GNU General Public License as published by | |
6 # the Free Software Foundation, either version 3 of the License, or | |
7 # (at your option) any later version. | |
8 # | |
9 # This program is distributed in the hope that it will be useful, | |
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 # GNU General Public License for more details. | |
13 # | |
14 # You should have received a copy of the GNU General Public License | |
15 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | |
17 import argparse | |
18 import calendar | |
19 from functools import partial, reduce | |
20 from glob import glob | |
21 import json | |
22 import os | |
7 | 23 from os.path import abspath, dirname, isdir |
1 | 24 import shlex |
25 from subprocess import Popen, PIPE, STDOUT | |
26 import sys | |
27 from syslog import * | |
28 import time | |
21
3ecfeaba3c92
fix exception handling
Changaco <changaco ατ changaco δοτ net>
parents:
20
diff
changeset
|
29 import traceback |
1 | 30 |
31 import feedparser | |
32 | |
33 | |
34 # Constants | |
35 | |
36 log_levels = ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERR', 'CRIT', 'ALERT', 'EMERG'] | |
37 | |
38 gamin_events = { | |
39 1:'GAMChanged', 2:'GAMDeleted', 3:'GAMStartExecuting', 4:'GAMStopExecuting', | |
40 5:'GAMCreated', 6:'GAMMoved', 7:'GAMAcknowledge', 8:'GAMExists', 9:'GAMEndExist' | |
41 } | |
42 | |
43 | |
44 # Generic utils | |
45 | |
46 concat = lambda l: reduce(list.__add__, l, []) | |
47 | |
48 def dict_append(d, k, v): | |
49 d.setdefault(k, []) | |
50 d[k].append(v) | |
51 | |
52 | |
53 # Logging | |
54 | |
55 def log(*args): | |
56 if len(args) == 1: | |
57 priority, msg = LOG_INFO, args[0] | |
58 elif len(args) == 2: | |
59 priority, msg = args | |
60 else: | |
61 return | |
62 priority = 7 - priority | |
63 if priority < global_args.log_level: | |
64 return | |
15
68a9b24a182a
encode log messages in UTF8 before passing them to syslog
Changaco <changaco ατ changaco δοτ net>
parents:
14
diff
changeset
|
65 if isinstance(msg, unicode): |
68a9b24a182a
encode log messages in UTF8 before passing them to syslog
Changaco <changaco ατ changaco δοτ net>
parents:
14
diff
changeset
|
66 msg = msg.encode('utf8') |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
67 try: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
68 if global_args.fork: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
69 syslog(priority, msg) |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
70 else: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
71 sys.stderr.write(log_levels[priority]+': '+msg+'\n') |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
72 except: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
73 exit(1) |
1 | 74 |
75 def ignore_event(path, event): | |
76 log(LOG_DEBUG, 'ignoring event '+gamin_events.get(event, str(event))+' on '+path) | |
77 | |
78 | |
79 # Config parsing | |
80 | |
81 def parse_config_file(config_fd): | |
82 feeds_paths = config_to_feed_paths_to_commands[config_fd.name] = {} | |
11 | 83 cmd = [] |
12
ddbbd4ee5d47
log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents:
11
diff
changeset
|
84 log('parsing config file '+config_fd.name) |
1 | 85 config_fd.seek(0) |
86 for i, line in enumerate(config_fd): | |
87 line = line.strip() | |
88 if len(line) == 0 or line[0] == '#': | |
89 continue | |
90 if line[0] == '%': | |
11 | 91 if isinstance(cmd, str): |
92 cmd = [] | |
93 cmd.append(line[1:].rstrip(';')) | |
94 elif not cmd: | |
1 | 95 log(LOG_ERR, 'missing command in file '+config_fd.name+' before line '+str(i)) |
96 return | |
97 else: | |
11 | 98 cmd = '; '.join(cmd) |
1 | 99 for feed_path in glob(line): |
100 feed_path = abspath(feed_path) | |
101 dict_append(feeds_paths, feed_path, cmd) | |
102 if not feed_path in path_to_feed_fd: | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
103 monitor.watch_file(feed_path, handler(handle_feed_change)) |
12
ddbbd4ee5d47
log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents:
11
diff
changeset
|
104 log('now watching '+feed_path) |
1 | 105 |
106 | |
107 # Gamin callbacks | |
108 | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
109 def handler(f): |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
110 def g(path, event): |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
111 try: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
112 f(path, event) |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
113 except: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
114 log(LOG_CRIT, traceback.format_exc()) |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
115 return g |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
116 |
1 | 117 def handle_config_change(path, event): |
118 path = abspath(path) | |
119 if os.path.isdir(path): | |
120 ignore_event(path, event) | |
121 elif not path in path_to_config_fd: | |
122 open_config(path, event) | |
123 elif event in [gamin.GAMChanged, gamin.GAMDeleted]: | |
124 update_config(path, event) | |
125 else: | |
126 ignore_event(path, event) | |
127 | |
128 def open_config(path, event): | |
129 if event in [gamin.GAMCreated, gamin.GAMExists]: | |
130 if (not path.endswith('.conf') or path[0] == '.') and not hasattr(global_args.config, 'read'): | |
131 return log('ignoring '+path+' (not a valid config file name)') | |
132 try: | |
133 config_fd = open(path) | |
134 except IOError as e: | |
135 return log('failed to open "'+line+'" '+str(e)) | |
136 path_to_config_fd[path] = config_fd | |
137 parse_config_file(config_fd) | |
138 else: | |
139 ignore_event(path, event) | |
140 | |
141 def update_config(path, event): | |
10
89f581ebf4b2
fix update_config function
Changaco <changaco ατ changaco δοτ net>
parents:
7
diff
changeset
|
142 feeds_paths = set(concat(d.keys() for d in config_to_feed_paths_to_commands.values())) |
1 | 143 if event == gamin.GAMChanged: |
144 parse_config_file(path_to_config_fd[path]) | |
145 elif event == gamin.GAMDeleted: | |
146 log('removing actions from deleted config file '+config_fd.name) | |
147 config_to_feed_paths_to_commands.pop(path) | |
148 path_to_config_fd.pop(path).close() | |
10
89f581ebf4b2
fix update_config function
Changaco <changaco ατ changaco δοτ net>
parents:
7
diff
changeset
|
149 new_feeds_paths = set(concat(d.keys() for d in config_to_feed_paths_to_commands.values())) |
1 | 150 for feed_path in feeds_paths.difference(new_feeds_paths): |
151 monitor.stop_watch(feed_path) | |
12
ddbbd4ee5d47
log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents:
11
diff
changeset
|
152 log('stopped watching '+feed_path) |
1 | 153 if feed_path in path_to_feed_fd: |
154 path_to_feed_fd.pop(feed_path).close() | |
155 | |
156 def handle_feed_change(path, event): | |
157 if path not in path_to_feed_fd: | |
158 if event in [gamin.GAMCreated, gamin.GAMExists, gamin.GAMChanged]: | |
159 try: | |
160 feed_fd = path_to_feed_fd[path] = open(path) | |
161 except IOError as e: | |
162 return log('failed to open "'+path+'": '+str(e)) | |
163 feed = feedparser.parse(feed_fd.read()) | |
164 handle_feed_change(path, gamin.GAMChanged) | |
165 else: | |
166 ignore_event(path, event) | |
167 elif event == gamin.GAMChanged: | |
168 feed_fd = path_to_feed_fd[path] | |
169 feed_fd.seek(0) | |
170 feed = feedparser.parse(feed_fd.read()) | |
23
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
171 i = 0 |
1 | 172 for entry in reversed(feed.entries): |
173 if entry.id in state['id_cache'].get(feed_fd.name, []) or \ | |
174 not global_args.flood and calendar.timegm(entry.published_parsed) < time.time() - 86400: | |
175 continue | |
23
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
176 i += 1 |
1 | 177 for feed_path_to_commands in config_to_feed_paths_to_commands.values(): |
178 for cmd in feed_path_to_commands.get(path, []): | |
179 run_command(format_cmd(cmd, feed=feed.feed, entry=entry), entry.content[0].value) | |
180 state['id_cache'][feed_fd.name] = [entry.id for entry in feed.entries] | |
181 save_state() | |
23
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
182 if i == 0: |
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
183 log(LOG_DEBUG, 'received GAMChanged event on "%s" but no new entry was found' % path) |
1 | 184 elif event == gamin.GAMDeleted: |
185 path_to_feed_fd.pop(path).close() | |
186 else: | |
187 ignore_event(path, event) | |
188 | |
189 def save_state(): | |
190 global_args.state_file.truncate(0) | |
191 json.dump(state, global_args.state_file) | |
192 global_args.state_file.flush() | |
193 | |
194 | |
195 # Commands utils | |
196 | |
197 def format_cmd(cmd, **kwargs): | |
198 """The safe equivalent of str.format() for shell commands, meaning interpolated variables can't do shell injections (I hope).""" | |
199 r = u'' | |
200 for arg in shlex.split(cmd.encode('utf8')): | |
201 a = arg.decode('utf8') | |
202 b = a.format(**kwargs) | |
203 if a != b: | |
204 r += u" '" + b.replace(u"'", u'\'"\'"\'') + u"'" | |
205 else: | |
206 r += u' ' + arg | |
207 return r.lstrip() | |
208 | |
209 def run_command(cmd, input): | |
210 p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True) | |
211 output = p.communicate(input.encode('utf8'))[0].decode('utf8') | |
212 if p.returncode != 0: | |
213 log(LOG_ERR, 'command failed: '+cmd+'\n'+output) | |
214 else: | |
215 log(LOG_INFO, 'successfully executed '+cmd) | |
216 log(LOG_DEBUG, '===== output:\n'+output) | |
217 | |
218 | |
219 # Argparse utils | |
220 | |
221 def AbsPath(next_type=None): | |
222 def f(s): | |
223 p = abspath(s) | |
224 if next_type is not None: | |
225 return next_type(p) | |
226 else: | |
227 return p | |
228 return f | |
229 | |
230 class Apply(argparse.Action): | |
231 def __init__(self, f, *args, **kwargs): | |
232 super(self.__class__, self).__init__(**kwargs) | |
233 self.f = f | |
234 def __call__(self, parser, namespace, values, option_string=None): | |
235 setattr(namespace, self.dest, self.f(values[0])) | |
236 | |
6
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
237 def MakeDirs(next_type=None): |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
238 def f(s): |
7 | 239 d = dirname(s) |
14
5c26fc1adbac
minor fix in automatic directory creation
Changaco <changaco ατ changaco δοτ net>
parents:
12
diff
changeset
|
240 if d and not isdir(d): |
6
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
241 try: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
242 os.makedirs(d) |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
243 except OSError as e: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
244 raise argparse.ArgumentTypeError(str(e)) |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
245 if next_type is not None: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
246 return next_type(s) |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
247 else: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
248 return s |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
249 return f |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
250 |
1 | 251 def Directory(s): |
252 try: | |
253 os.listdir(s) | |
254 return s | |
255 except OSError as e: | |
256 raise argparse.ArgumentTypeError(str(e)) | |
257 | |
258 def File(flags): | |
259 def f(s): | |
260 try: | |
261 return os.fdopen(os.open(s, flags), 'w') | |
262 except OSError as e: | |
263 raise argparse.ArgumentTypeError(str(e)) | |
264 return f | |
265 | |
266 class First(argparse.Action): | |
267 def __call__(self, parser, namespace, values, option_string=None): | |
268 setattr(namespace, self.dest, values[0]) | |
269 | |
270 def FirstOf(*types, **kwargs): | |
271 kwargs.setdefault('error', 'argument "{}" is not valid') | |
272 def f(s): | |
273 for t in types: | |
274 try: | |
275 return t(s) | |
276 except: | |
277 pass | |
278 raise argparse.ArgumentTypeError(error.format(s)) | |
279 return f | |
280 | |
281 | |
282 # Main | |
283 | |
284 if __name__ == '__main__': | |
285 | |
286 p = argparse.ArgumentParser() | |
287 p.add_argument('config', type=FirstOf(AbsPath(argparse.FileType('r')), AbsPath(Directory), error='"{}" is neither a file nor a directory'), help='either a file or a directory') | |
6
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
288 p.add_argument('state_file', type=MakeDirs(argparse.FileType('a+')), help='e.g. /var/lib/feed-push/state') |
1 | 289 p.add_argument('--flood', default=False, action='store_true', help='push all articles on startup instead of ignoring the ones older than 24h (useful for debugging)') |
16
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
290 p.add_argument('--fork', metavar='pid-file', nargs=1, type=File(os.O_WRONLY|os.O_CREAT|os.O_EXCL), action=First, help='daemonize and log to syslog') |
1 | 291 p.add_argument('--log-level', nargs=1, default=1, choices=log_levels, action=partial(Apply, log_levels.index), help='default is INFO') |
292 global_args = p.parse_args() | |
293 | |
294 if global_args.fork: | |
295 pid = os.fork() | |
296 if pid != 0: | |
16
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
297 exit(0) |
19 | 298 os.setsid() |
16
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
299 pid = os.fork() |
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
300 if pid != 0: |
1 | 301 global_args.fork.write(str(pid)) |
302 exit(0) | |
303 openlog(facility=LOG_DAEMON) | |
17
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
304 null = open('/dev/null', 'r+') |
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
305 for f in [sys.stdin, sys.stdout, sys.stderr]: |
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
306 f.flush() |
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
307 os.dup2(null.fileno(), f.fileno()) |
1 | 308 |
309 state = {'id_cache': {}} | |
310 saved_state = global_args.state_file.read().strip() | |
311 if len(saved_state) > 0: | |
312 state.update(json.loads(saved_state)) | |
313 del saved_state | |
314 | |
22
b763ca084088
import gamin in __main__
Changaco <changaco ατ changaco δοτ net>
parents:
21
diff
changeset
|
315 import gamin |
1 | 316 monitor = gamin.WatchMonitor() |
317 path_to_feed_fd = {} | |
318 path_to_config_fd = {} | |
319 config_to_feed_paths_to_commands = {} | |
320 if hasattr(global_args.config, 'read'): | |
321 os.chdir(os.path.dirname(global_args.config.name)) | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
322 monitor.watch_file(global_args.config.name, handler(handle_config_change)) |
1 | 323 else: |
324 os.chdir(global_args.config) | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
325 monitor.watch_directory(global_args.config, handler(handle_config_change)) |
1 | 326 |
20
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
327 if global_args.fork: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
328 while True: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
329 monitor.handle_one_event() |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
330 else: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
331 try: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
332 while True: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
333 monitor.handle_events() |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
334 time.sleep(1) |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
335 except KeyboardInterrupt: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
336 exit(0) |