#!/usr/bin/python3 import os import datetime import random import re import lxml.etree as etree import filer from nitpo import Nitpo from profile import Profile class Mailman(Nitpo): def __init__(self, do_verbose=True): """work with mailman data""" super().__init__() self.check_conf('folders', 'profile') self.reasons = {} self.reasons['disabled address'] = 1 self.reasons['web confirmation'] = 1 self.reasons['bin/remove_members'] = 1 self.reasons['via the member options page'] = 1 self.reasons['email confirmation'] = 1 self.reasons['member mgt page'] = 1 self.reasons['admin mass sub'] = 1 self.reasons['via web confirmation'] = 1 self.reasons['via email confirmation'] = 1 self.reasons['via admin approval'] = 1 self.reasons['bin/add_members'] = 1 self.reasons['admin mass unsub'] = 1 self.re_namd = re.compile(r'(.*) <([^ >]+@[^ ]+)>$') self.re_emad = re.compile(r'<*([^ ]+@[^ ]+)>*$') self.re_dumd = re.compile(r'\s*"([^@]+@[^@]+)"\s+<\1>\s*$') self.re_chan = re.compile(r'changed member address from (\S+) to (\S+)+$') self.events = [] self.event = {} self.do_verbose = do_verbose # def watch(self): # self.read_log() # random.shuffle(self.events) # for event in self.events: # self.insert(event) # quit() def read_log(self, log_fufi, do_verbose=False): if not os.path.isfile(log_fufi): return [] log_file = open(log_fufi, 'r', encoding='iso-8859-1') while line := log_file.readline(): self.event = {} self.log_line(line.rstrip(), do_verbose=do_verbose) log_file.close() return self.events def log_line(self, line, do_verbose=False): """works on a line""" if ' pending ' in line: #if do_verbose # print(line) return # # this is a duplication from a previous del if ' auto-unsubscribed ' in line: #if do_verbose # print('DISCARD ' + line) return if do_verbose: print(line) date = line[0:20] rest = line[21:] time = datetime.datetime.strptime(date, "%b %d %H:%M:%S %Y") time = datetime.datetime.strftime(time, "%Y-%m-%dT%H:%M:%SZ") self.event['time'] = time repcode = rest.split(' ')[1:2][0][0:7] if repcode[0:4] != 'nep-' or repcode == 'nep-tes': return self.event['repcode'] = repcode if 'name' in self.event: del self.event['name'] if 'reason' in self.event: del self.event['reason'] rest = rest.split(repcode + ": ")[1:][0] verb = rest.partition(' ')[0] if verb == 'pending': return None if verb == 'deleted': self.event['act'] = 'del' self.del_log(rest) if do_verbose: print(self.event) self.events.append(self.event) return if 'auto-unsubscribed' in line: parts = rest.partition(' auto-unsubscribed ') emad = parts[0] self.event['act'] = 'del' self.event['emad'] = emad if do_verbose: print(self.event) self.events.append(self.event) return if rest.startswith('new '): self.event['act'] = 'add' rest = rest[4:] self.add_log(rest) if do_verbose: print(self.event) self.events.append(self.event) return if changed := self.re_chan.match(rest): old = changed.group(1) new = changed.group(2) self.event = {} # # for both self.event['repcode'] = repcode self.event['time'] = time # # for each self.event['act'] = 'add' self.event['emad'] = new self.events.append(self.event) if do_verbose: print(self.event) self.event['act'] = 'del' self.event['emad'] = old if do_verbose: print(self.event) self.events.append(self.event) return raise Exception(line) def add_log(self, line): if '(digest)' in line: line = line.replace('(digest) ', '') parts = line.partition(',') reason = parts[2].lstrip() if reason not in self.reasons: raise Exception(f'unknown reason {reason} in {line}') if reason.startswith('via '): reason = reason[4:] self.event['via'] = reason orig = parts[0].strip() if res := self.re_dumd.match(orig): emad = res.group(1).strip() self.event['emad'] = emad return if res := self.re_namd.match(orig): name = res.group(1).strip() if name.startswith('"'): name = name[1:] if name.endswith('"'): name = name[:-1] emad = res.group(2).strip() self.event['name'] = name self.event['emad'] = emad return if res := self.re_emad.match(orig): emad = res.group(1) self.event['emad'] = emad return raise Exception(f"UNDETECTED {orig}") def del_log(self, line): parts = line.partition(';') reason = parts[2].lstrip() if reason not in self.reasons: raise Exception(f'unknown reason {reason} in {line}') emad = parts[0] emad = emad.replace('deleted ', '') self.event['emad'] = emad self.event['reason'] = reason #print('reason: ' + reason) def insert(self, event): p = Profile() root = p.load(event['emad']) print(etree.tostring(root).decode()) print(event) def close_date(d1, d2): t1 = datetime.datetime.strptime(d1, "%Y-%m-%dT%H:%M:%SZ") t2 = datetime.datetime.strptime(d2, "%Y-%m-%dT%H:%M:%SZ") if abs((t2 - t1).days()) <= 1: return True return False