#!/usr/bin/python3 import os import datetime import re import lxml.etree as etree from nitpo import Nitpo from profile import Profile class Mailman(Nitpo): def __init__(self, do_verbose=True): """work with mailman data""" super().__init__() self.check_conf('folders', 'profile') self.vias = {} self.vias['disabled address'] = 1 self.vias['web confirmation'] = 1 self.vias['bin/remove_members'] = 1 self.vias['via the member options page'] = 1 self.vias['email confirmation'] = 1 self.vias['member mgt page'] = 1 self.vias['admin mass sub'] = 1 self.vias['via web confirmation'] = 1 self.vias['via email confirmation'] = 1 self.vias['via admin approval'] = 1 self.vias['bin/add_members'] = 1 self.vias['admin mass unsub'] = 1 self.re_namd = re.compile(r'(.*) <([^ >]+@[^ ]+)>$') self.re_emad = re.compile(r'<*([^ ]+@[^ ]+)>*$') self.re_dumd = re.compile(r'\s*"([^@]+@[^@]+)"\s+<\1>\s*$') self.re_chan = re.compile(r'changed member address from (\S+) to (\S+)+$') # # inverse names is namas self.re_invna = re.compile(r'"([A-z]+), ([A-z]+)" (<[^ >]+@[^ ]+>)') self.bad_repcodes = ('nep-all', 'nep-xxx', 'nep-yyy', 'nep-zzz') self.events = [] self.event = {} self.do_verbose = do_verbose def read_log(self, log_fufi, do_verbose=False): if not os.path.isfile(log_fufi): return [] log_file = open(log_fufi, 'r', encoding='iso-8859-1') while line := log_file.readline(): self.event = {} self.log_line(line.rstrip(), do_verbose=do_verbose) log_file.close() return self.events def log_line(self, line, do_verbose=False): """works on a line""" # restructure for "Gilju, Kim" line = self.re_invna.sub('"\\2 \\1" \\3', line) if ' pending ' in line: #if do_verbose # print(line) return # # this is a duplication from a previous del if ' auto-unsubscribed ' in line: #if do_verbose # print('DISCARD ' + line) return if do_verbose: print(line) date = line[0:20] rest = line[21:] time = datetime.datetime.strptime(date, "%b %d %H:%M:%S %Y") time = datetime.datetime.strftime(time, "%Y-%m-%dT%H:%M:%SZ") self.event['time'] = time repcode = rest.split(' ')[1:2][0][0:7] if repcode[0:4] != 'nep-': return if repcode in self.bad_repcodes: return self.event['repcode'] = repcode if 'name' in self.event: del self.event['name'] if 'via' in self.event: del self.event['via'] rest = rest.split(repcode + ": ")[1:][0] verb = rest.partition(' ')[0] if verb == 'pending': return None if verb == 'deleted': self.event['act'] = 'del' self.del_log(rest) if do_verbose: print(self.event) self.events.append(self.event) return if 'auto-unsubscribed' in line: parts = rest.partition(' auto-unsubscribed ') emad = parts[0] self.event['act'] = 'del' self.event['emad'] = emad if do_verbose: print(self.event) self.events.append(self.event) return if rest.startswith('new '): self.event['act'] = 'add' rest = rest[4:] self.add_log(rest) if do_verbose: print(self.event) self.events.append(self.event) return if changed := self.re_chan.match(rest): old = changed.group(1) new = changed.group(2) for emad in (old, new): self.event = {} self.event['time'] = time self.event['repcode'] = repcode if emad == old: self.event['act'] = 'del' self.event['emad'] = old if emad == new: self.event['act'] = 'add' self.event['emad'] = new self.event['change_emad'] = 1 self.events.append(self.event) if do_verbose: print(self.event) return raise Exception(line) def add_log(self, line): if '(digest)' in line: line = line.replace('(digest) ', '') # # fix an annoying special case line = line.replace(', PhD', '') parts = line.partition(',') via = parts[2].lstrip() if via not in self.vias: raise Exception(f'unknown via {via} in {line}') if via.startswith('via '): via = via[4:] self.event['via'] = via orig = parts[0].strip() if res := self.re_dumd.match(orig): emad = res.group(1).strip() self.event['emad'] = emad return if res := self.re_namd.match(orig): name = res.group(1).strip() if name.startswith('"'): name = name[1:] if name.endswith('"'): name = name[:-1] emad = res.group(2).strip() self.event['name'] = name self.event['emad'] = emad return if res := self.re_emad.match(orig): emad = res.group(1) self.event['emad'] = emad return raise Exception(f"UNDETECTED {orig}") def del_log(self, line): parts = line.partition(';') via = parts[2].lstrip() if via not in self.vias: raise Exception(f'unknown via {via} in {line}') emad = parts[0] emad = emad.replace('deleted ', '') self.event['emad'] = emad self.event['via'] = via #print('via: ' + via) def insert(self, event): p = Profile() root = p.load(event['emad']) print(etree.tostring(root).decode()) print(event) # def close_date(d1, d2): # t1 = datetime.datetime.strptime(d1, "%Y-%m-%dT%H:%M:%SZ") # t2 = datetime.datetime.strptime(d2, "%Y-%m-%dT%H:%M:%SZ") # if abs((t2 - t1).days()) <= 1: # return True # return False