Mercurial > feed-push
annotate feed-push @ 29:ab4ef352ae36 draft
change level of a log message
Signed-off-by: Changaco <changaco ατ changaco δοτ net>
author | Changaco <changaco ατ changaco δοτ net> |
---|---|
date | Sat, 04 Aug 2012 21:29:44 +0200 |
parents | 022c785c98bd |
children | 7e935b48f53a |
rev | line source |
---|---|
1 | 1 #!/usr/bin/env python2 |
2 # -*- coding: utf-8 -*- | |
3 | |
4 # This program is free software: you can redistribute it and/or modify | |
5 # it under the terms of the GNU General Public License as published by | |
6 # the Free Software Foundation, either version 3 of the License, or | |
7 # (at your option) any later version. | |
8 # | |
9 # This program is distributed in the hope that it will be useful, | |
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 # GNU General Public License for more details. | |
13 # | |
14 # You should have received a copy of the GNU General Public License | |
15 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | |
17 import argparse | |
18 import calendar | |
19 from functools import partial, reduce | |
20 from glob import glob | |
21 import json | |
22 import os | |
7 | 23 from os.path import abspath, dirname, isdir |
1 | 24 import shlex |
25 from subprocess import Popen, PIPE, STDOUT | |
26 import sys | |
27 from syslog import * | |
28 import time | |
21
3ecfeaba3c92
fix exception handling
Changaco <changaco ατ changaco δοτ net>
parents:
20
diff
changeset
|
29 import traceback |
1 | 30 |
31 import feedparser | |
32 | |
33 | |
34 # Constants | |
35 | |
36 log_levels = ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERR', 'CRIT', 'ALERT', 'EMERG'] | |
37 | |
38 gamin_events = { | |
39 1:'GAMChanged', 2:'GAMDeleted', 3:'GAMStartExecuting', 4:'GAMStopExecuting', | |
40 5:'GAMCreated', 6:'GAMMoved', 7:'GAMAcknowledge', 8:'GAMExists', 9:'GAMEndExist' | |
41 } | |
42 | |
43 | |
44 # Generic utils | |
45 | |
46 concat = lambda l: reduce(list.__add__, l, []) | |
47 | |
48 def dict_append(d, k, v): | |
49 d.setdefault(k, []) | |
50 d[k].append(v) | |
51 | |
52 | |
53 # Logging | |
54 | |
55 def log(*args): | |
56 if len(args) == 1: | |
57 priority, msg = LOG_INFO, args[0] | |
58 elif len(args) == 2: | |
59 priority, msg = args | |
60 else: | |
61 return | |
62 priority = 7 - priority | |
63 if priority < global_args.log_level: | |
64 return | |
15
68a9b24a182a
encode log messages in UTF8 before passing them to syslog
Changaco <changaco ατ changaco δοτ net>
parents:
14
diff
changeset
|
65 if isinstance(msg, unicode): |
68a9b24a182a
encode log messages in UTF8 before passing them to syslog
Changaco <changaco ατ changaco δοτ net>
parents:
14
diff
changeset
|
66 msg = msg.encode('utf8') |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
67 try: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
68 if global_args.fork: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
69 syslog(priority, msg) |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
70 else: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
71 sys.stderr.write(log_levels[priority]+': '+msg+'\n') |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
72 except: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
73 exit(1) |
1 | 74 |
75 def ignore_event(path, event): | |
76 log(LOG_DEBUG, 'ignoring event '+gamin_events.get(event, str(event))+' on '+path) | |
77 | |
78 | |
79 # Config parsing | |
80 | |
81 def parse_config_file(config_fd): | |
82 feeds_paths = config_to_feed_paths_to_commands[config_fd.name] = {} | |
11 | 83 cmd = [] |
12
ddbbd4ee5d47
log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents:
11
diff
changeset
|
84 log('parsing config file '+config_fd.name) |
1 | 85 config_fd.seek(0) |
86 for i, line in enumerate(config_fd): | |
87 line = line.strip() | |
88 if len(line) == 0 or line[0] == '#': | |
89 continue | |
90 if line[0] == '%': | |
11 | 91 if isinstance(cmd, str): |
92 cmd = [] | |
93 cmd.append(line[1:].rstrip(';')) | |
94 elif not cmd: | |
1 | 95 log(LOG_ERR, 'missing command in file '+config_fd.name+' before line '+str(i)) |
96 return | |
97 else: | |
11 | 98 cmd = '; '.join(cmd) |
1 | 99 for feed_path in glob(line): |
100 feed_path = abspath(feed_path) | |
101 dict_append(feeds_paths, feed_path, cmd) | |
102 if not feed_path in path_to_feed_fd: | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
103 monitor.watch_file(feed_path, handler(handle_feed_change)) |
12
ddbbd4ee5d47
log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents:
11
diff
changeset
|
104 log('now watching '+feed_path) |
1 | 105 |
106 | |
107 # Gamin callbacks | |
108 | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
109 def handler(f): |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
110 def g(path, event): |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
111 try: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
112 f(path, event) |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
113 except: |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
114 log(LOG_CRIT, traceback.format_exc()) |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
115 return g |
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
116 |
1 | 117 def handle_config_change(path, event): |
118 path = abspath(path) | |
119 if os.path.isdir(path): | |
120 ignore_event(path, event) | |
121 elif not path in path_to_config_fd: | |
122 open_config(path, event) | |
123 elif event in [gamin.GAMChanged, gamin.GAMDeleted]: | |
124 update_config(path, event) | |
125 else: | |
126 ignore_event(path, event) | |
127 | |
128 def open_config(path, event): | |
129 if event in [gamin.GAMCreated, gamin.GAMExists]: | |
130 if (not path.endswith('.conf') or path[0] == '.') and not hasattr(global_args.config, 'read'): | |
131 return log('ignoring '+path+' (not a valid config file name)') | |
132 try: | |
28 | 133 config_fd = path_to_config_fd[path] = open(path) |
1 | 134 except IOError as e: |
135 return log('failed to open "'+line+'" '+str(e)) | |
136 parse_config_file(config_fd) | |
137 else: | |
138 ignore_event(path, event) | |
139 | |
140 def update_config(path, event): | |
10
89f581ebf4b2
fix update_config function
Changaco <changaco ατ changaco δοτ net>
parents:
7
diff
changeset
|
141 feeds_paths = set(concat(d.keys() for d in config_to_feed_paths_to_commands.values())) |
1 | 142 if event == gamin.GAMChanged: |
143 parse_config_file(path_to_config_fd[path]) | |
144 elif event == gamin.GAMDeleted: | |
25 | 145 log('removing actions from deleted config file '+path) |
1 | 146 config_to_feed_paths_to_commands.pop(path) |
147 path_to_config_fd.pop(path).close() | |
10
89f581ebf4b2
fix update_config function
Changaco <changaco ατ changaco δοτ net>
parents:
7
diff
changeset
|
148 new_feeds_paths = set(concat(d.keys() for d in config_to_feed_paths_to_commands.values())) |
1 | 149 for feed_path in feeds_paths.difference(new_feeds_paths): |
150 monitor.stop_watch(feed_path) | |
12
ddbbd4ee5d47
log watched/unwatched feed paths
Changaco <changaco ατ changaco δοτ net>
parents:
11
diff
changeset
|
151 log('stopped watching '+feed_path) |
1 | 152 if feed_path in path_to_feed_fd: |
153 path_to_feed_fd.pop(feed_path).close() | |
154 | |
155 def handle_feed_change(path, event): | |
156 if path not in path_to_feed_fd: | |
157 if event in [gamin.GAMCreated, gamin.GAMExists, gamin.GAMChanged]: | |
158 try: | |
24
96e2d5ffcd23
remove double feed parsing
Changaco <changaco ατ changaco δοτ net>
parents:
23
diff
changeset
|
159 path_to_feed_fd[path] = open(path) |
1 | 160 except IOError as e: |
161 return log('failed to open "'+path+'": '+str(e)) | |
162 handle_feed_change(path, gamin.GAMChanged) | |
163 else: | |
164 ignore_event(path, event) | |
165 elif event == gamin.GAMChanged: | |
166 feed_fd = path_to_feed_fd[path] | |
167 feed_fd.seek(0) | |
168 feed = feedparser.parse(feed_fd.read()) | |
23
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
169 i = 0 |
1 | 170 for entry in reversed(feed.entries): |
171 if entry.id in state['id_cache'].get(feed_fd.name, []) or \ | |
172 not global_args.flood and calendar.timegm(entry.published_parsed) < time.time() - 86400: | |
173 continue | |
23
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
174 i += 1 |
1 | 175 for feed_path_to_commands in config_to_feed_paths_to_commands.values(): |
176 for cmd in feed_path_to_commands.get(path, []): | |
177 run_command(format_cmd(cmd, feed=feed.feed, entry=entry), entry.content[0].value) | |
178 state['id_cache'][feed_fd.name] = [entry.id for entry in feed.entries] | |
179 save_state() | |
23
5bb7c2939da0
add some debug logging
Changaco <changaco ατ changaco δοτ net>
parents:
22
diff
changeset
|
180 if i == 0: |
29
ab4ef352ae36
change level of a log message
Changaco <changaco ατ changaco δοτ net>
parents:
28
diff
changeset
|
181 log('no new entry found after GAMChanged event on %s' % path) |
1 | 182 elif event == gamin.GAMDeleted: |
183 path_to_feed_fd.pop(path).close() | |
184 else: | |
185 ignore_event(path, event) | |
186 | |
187 def save_state(): | |
188 global_args.state_file.truncate(0) | |
189 json.dump(state, global_args.state_file) | |
190 global_args.state_file.flush() | |
191 | |
192 | |
193 # Commands utils | |
194 | |
195 def format_cmd(cmd, **kwargs): | |
196 """The safe equivalent of str.format() for shell commands, meaning interpolated variables can't do shell injections (I hope).""" | |
197 r = u'' | |
198 for arg in shlex.split(cmd.encode('utf8')): | |
199 a = arg.decode('utf8') | |
200 b = a.format(**kwargs) | |
201 if a != b: | |
202 r += u" '" + b.replace(u"'", u'\'"\'"\'') + u"'" | |
203 else: | |
204 r += u' ' + arg | |
205 return r.lstrip() | |
206 | |
207 def run_command(cmd, input): | |
208 p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True) | |
209 output = p.communicate(input.encode('utf8'))[0].decode('utf8') | |
210 if p.returncode != 0: | |
211 log(LOG_ERR, 'command failed: '+cmd+'\n'+output) | |
212 else: | |
213 log(LOG_INFO, 'successfully executed '+cmd) | |
214 log(LOG_DEBUG, '===== output:\n'+output) | |
215 | |
216 | |
217 # Argparse utils | |
218 | |
219 def AbsPath(next_type=None): | |
220 def f(s): | |
221 p = abspath(s) | |
222 if next_type is not None: | |
223 return next_type(p) | |
224 else: | |
225 return p | |
226 return f | |
227 | |
228 class Apply(argparse.Action): | |
229 def __init__(self, f, *args, **kwargs): | |
230 super(self.__class__, self).__init__(**kwargs) | |
231 self.f = f | |
232 def __call__(self, parser, namespace, values, option_string=None): | |
233 setattr(namespace, self.dest, self.f(values[0])) | |
234 | |
6
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
235 def MakeDirs(next_type=None): |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
236 def f(s): |
7 | 237 d = dirname(s) |
14
5c26fc1adbac
minor fix in automatic directory creation
Changaco <changaco ατ changaco δοτ net>
parents:
12
diff
changeset
|
238 if d and not isdir(d): |
6
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
239 try: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
240 os.makedirs(d) |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
241 except OSError as e: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
242 raise argparse.ArgumentTypeError(str(e)) |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
243 if next_type is not None: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
244 return next_type(s) |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
245 else: |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
246 return s |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
247 return f |
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
248 |
1 | 249 def Directory(s): |
250 try: | |
251 os.listdir(s) | |
252 return s | |
253 except OSError as e: | |
254 raise argparse.ArgumentTypeError(str(e)) | |
255 | |
256 def File(flags): | |
257 def f(s): | |
258 try: | |
259 return os.fdopen(os.open(s, flags), 'w') | |
260 except OSError as e: | |
261 raise argparse.ArgumentTypeError(str(e)) | |
262 return f | |
263 | |
264 class First(argparse.Action): | |
265 def __call__(self, parser, namespace, values, option_string=None): | |
266 setattr(namespace, self.dest, values[0]) | |
267 | |
268 def FirstOf(*types, **kwargs): | |
269 kwargs.setdefault('error', 'argument "{}" is not valid') | |
270 def f(s): | |
271 for t in types: | |
272 try: | |
273 return t(s) | |
274 except: | |
275 pass | |
276 raise argparse.ArgumentTypeError(error.format(s)) | |
277 return f | |
278 | |
279 | |
280 # Main | |
281 | |
282 if __name__ == '__main__': | |
283 | |
284 p = argparse.ArgumentParser() | |
285 p.add_argument('config', type=FirstOf(AbsPath(argparse.FileType('r')), AbsPath(Directory), error='"{}" is neither a file nor a directory'), help='either a file or a directory') | |
6
455cd8c78862
create state dir if it doesn't exist
Changaco <changaco ατ changaco δοτ net>
parents:
1
diff
changeset
|
286 p.add_argument('state_file', type=MakeDirs(argparse.FileType('a+')), help='e.g. /var/lib/feed-push/state') |
1 | 287 p.add_argument('--flood', default=False, action='store_true', help='push all articles on startup instead of ignoring the ones older than 24h (useful for debugging)') |
16
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
288 p.add_argument('--fork', metavar='pid-file', nargs=1, type=File(os.O_WRONLY|os.O_CREAT|os.O_EXCL), action=First, help='daemonize and log to syslog') |
1 | 289 p.add_argument('--log-level', nargs=1, default=1, choices=log_levels, action=partial(Apply, log_levels.index), help='default is INFO') |
290 global_args = p.parse_args() | |
291 | |
292 if global_args.fork: | |
293 pid = os.fork() | |
294 if pid != 0: | |
16
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
295 exit(0) |
19 | 296 os.setsid() |
16
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
297 pid = os.fork() |
183f4f544987
merge --syslog into --fork and implement double forking
Changaco <changaco ατ changaco δοτ net>
parents:
15
diff
changeset
|
298 if pid != 0: |
1 | 299 global_args.fork.write(str(pid)) |
300 exit(0) | |
301 openlog(facility=LOG_DAEMON) | |
17
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
302 null = open('/dev/null', 'r+') |
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
303 for f in [sys.stdin, sys.stdout, sys.stderr]: |
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
304 f.flush() |
421a3416e768
close std{in,out,err} in daemon mode
Changaco <changaco ατ changaco δοτ net>
parents:
16
diff
changeset
|
305 os.dup2(null.fileno(), f.fileno()) |
1 | 306 |
307 state = {'id_cache': {}} | |
308 saved_state = global_args.state_file.read().strip() | |
309 if len(saved_state) > 0: | |
310 state.update(json.loads(saved_state)) | |
311 del saved_state | |
312 | |
22
b763ca084088
import gamin in __main__
Changaco <changaco ατ changaco δοτ net>
parents:
21
diff
changeset
|
313 import gamin |
1 | 314 monitor = gamin.WatchMonitor() |
315 path_to_feed_fd = {} | |
316 path_to_config_fd = {} | |
317 config_to_feed_paths_to_commands = {} | |
318 if hasattr(global_args.config, 'read'): | |
319 os.chdir(os.path.dirname(global_args.config.name)) | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
320 monitor.watch_file(global_args.config.name, handler(handle_config_change)) |
1 | 321 else: |
322 os.chdir(global_args.config) | |
18
868a339b19f8
log exceptions, exit when logging fails
Changaco <changaco ατ changaco δοτ net>
parents:
17
diff
changeset
|
323 monitor.watch_directory(global_args.config, handler(handle_config_change)) |
1 | 324 |
20
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
325 if global_args.fork: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
326 while True: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
327 monitor.handle_one_event() |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
328 else: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
329 try: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
330 while True: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
331 monitor.handle_events() |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
332 time.sleep(1) |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
333 except KeyboardInterrupt: |
94f3b22beec9
use polling in non-fork mode to catch KeyboardInterrupt
Changaco <changaco ατ changaco δοτ net>
parents:
19
diff
changeset
|
334 exit(0) |