diff options
Diffstat (limited to 'src')
47 files changed, 462 insertions, 1861 deletions
diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index c50169a7c..996493dc4 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -32,7 +32,8 @@ from mailman.core.i18n import _ from mailman.email.message import OwnerNotification from mailman.interfaces.bans import IBanManager from mailman.interfaces.member import ( - MemberRole, MembershipIsBannedError, NotAMemberError, SubscriptionEvent) + AlreadySubscribedError, MemberRole, MembershipIsBannedError, + NotAMemberError, SubscriptionEvent) from mailman.interfaces.usermanager import IUserManager from mailman.utilities.i18n import make from zope.component import getUtility @@ -96,15 +97,32 @@ def add_member(mlist, email, display_name, password, delivery_mode, language, member = mlist.subscribe(address, role) member.preferences.delivery_mode = delivery_mode else: - # The user exists and is linked to the address. + # The user exists and is linked to the case-insensitive address. + # We're looking for two versions of the email address, the case + # preserved version and the case insensitive version. We'll + # subscribe the version with matching case if it exists, otherwise + # we'll use one of the matching case-insensitively ones. It's + # undefined which one we pick. + case_preserved = None + case_insensitive = None for address in user.addresses: - if address.email == email: - break - else: - raise AssertionError( - 'User should have had linked address: {0}'.format(address)) - # Create the member and set the appropriate preferences. - member = mlist.subscribe(address, role) + if address.original_email == email: + case_preserved = address + if address.email == email.lower(): + case_insensitive = address + assert case_preserved is not None or case_insensitive is not None, ( + 'Could not find a linked address for: {}'.format(email)) + address = (case_preserved if case_preserved is not None + else case_insensitive) + # Create the member and set the appropriate preferences. It's + # possible we're subscribing the lower cased version of the address; + # if that's already subscribed re-issue the exception with the correct + # email address (i.e. the one passed in here). + try: + member = mlist.subscribe(address, role) + except AlreadySubscribedError as error: + raise AlreadySubscribedError( + error.fqdn_listname, email, error.role) member.preferences.preferred_language = language member.preferences.delivery_mode = delivery_mode return member diff --git a/src/mailman/app/tests/test_membership.py b/src/mailman/app/tests/test_membership.py index 9b42c21d6..cdf0641ea 100644 --- a/src/mailman/app/tests/test_membership.py +++ b/src/mailman/app/tests/test_membership.py @@ -166,6 +166,32 @@ class TestAddMember(unittest.TestCase): self.assertEqual(member_1.role, MemberRole.member) self.assertEqual(member_2.role, MemberRole.owner) + def test_add_member_with_mixed_case_email(self): + # LP: #1425359 - Mailman is case-perserving, case-insensitive. This + # test subscribes the lower case address and ensures the original + # mixed case address can't be subscribed. + email = 'APerson@example.com' + add_member(self._mlist, email.lower(), 'Ann Person', '123', + DeliveryMode.regular, system_preferences.preferred_language) + with self.assertRaises(AlreadySubscribedError) as cm: + add_member(self._mlist, email, 'Ann Person', '123', + DeliveryMode.regular, + system_preferences.preferred_language) + self.assertEqual(cm.exception.email, email) + + def test_add_member_with_lower_case_email(self): + # LP: #1425359 - Mailman is case-perserving, case-insensitive. This + # test subscribes the mixed case address and ensures the lower cased + # address can't be added. + email = 'APerson@example.com' + add_member(self._mlist, email, 'Ann Person', '123', + DeliveryMode.regular, system_preferences.preferred_language) + with self.assertRaises(AlreadySubscribedError) as cm: + add_member(self._mlist, email.lower(), 'Ann Person', '123', + DeliveryMode.regular, + system_preferences.preferred_language) + self.assertEqual(cm.exception.email, email.lower()) + class TestAddMemberPassword(unittest.TestCase): diff --git a/src/mailman/bin/bumpdigests.py b/src/mailman/bin/bumpdigests.py deleted file mode 100644 index f30772ca8..000000000 --- a/src/mailman/bin/bumpdigests.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import sys -import optparse - -from mailman import errors -from mailman import MailList -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - -# Work around known problems with some RedHat cron daemons -import signal -signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] [listname ...] - -Increment the digest volume number and reset the digest number to one. All -the lists named on the command line are bumped. If no list names are given, -all lists are bumped.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - return opts, args, parser - - - -def main(): - opts, args, parser = parseargs() - config.load(opts.config) - - listnames = set(args or config.list_manager.names) - if not listnames: - print(_('Nothing to do.')) - sys.exit(0) - - for listname in listnames: - try: - # Be sure the list is locked - mlist = MailList.MailList(listname) - except errors.MMListError: - parser.print_help() - print(_('No such list: $listname'), file=sys.stderr) - sys.exit(1) - try: - mlist.bump_digest_volume() - finally: - mlist.Save() - mlist.Unlock() - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/checkdbs.py b/src/mailman/bin/checkdbs.py deleted file mode 100644 index 61aa6b6f1..000000000 --- a/src/mailman/bin/checkdbs.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import sys -import time -import optparse - -from email.Charset import Charset - -from mailman import MailList -from mailman import Utils -from mailman.app.requests import handle_request -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.email.message import UserNotification -from mailman.initialize import initialize -from mailman.interfaces.requests import IListRequests, RequestType -from mailman.version import MAILMAN_VERSION - -# Work around known problems with some RedHat cron daemons -import signal -signal.signal(signal.SIGCHLD, signal.SIG_DFL) - -NL = u'\n' -now = time.time() - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Check for pending admin requests and mail the list owners if necessary.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print(_('Unexpected arguments'), file=sys.stderr) - sys.exit(1) - return opts, args, parser - - - -def pending_requests(mlist): - # Must return a byte string - lcset = mlist.preferred_language.charset - pending = [] - first = True - requestsdb = IListRequests(mlist) - for request in requestsdb.of_type(RequestType.subscription): - if first: - pending.append(_('Pending subscriptions:')) - first = False - key, data = requestsdb.get_request(request.id) - when = data['when'] - addr = data['addr'] - fullname = data['fullname'] - passwd = data['passwd'] - digest = data['digest'] - lang = data['lang'] - if fullname: - if isinstance(fullname, unicode): - fullname = fullname.encode(lcset, 'replace') - fullname = ' (%s)' % fullname - pending.append(' %s%s %s' % (addr, fullname, time.ctime(when))) - first = True - for request in requestsdb.of_type(RequestType.held_message): - if first: - pending.append(_('\nPending posts:')) - first = False - key, data = requestsdb.get_request(request.id) - when = data['when'] - sender = data['sender'] - subject = data['subject'] - reason = data['reason'] - text = data['text'] - msgdata = data['msgdata'] - subject = Utils.oneline(subject, lcset) - date = time.ctime(when) - reason = _(reason) - pending.append(_("""\ -From: $sender on $date -Subject: $subject -Cause: $reason""")) - pending.append('') - # Coerce all items in pending to a Unicode so we can join them - upending = [] - charset = mlist.preferred_language.charset - for s in pending: - if isinstance(s, unicode): - upending.append(s) - else: - upending.append(unicode(s, charset, 'replace')) - # Make sure that the text we return from here can be encoded to a byte - # string in the charset of the list's language. This could fail if for - # example, the request was pended while the list's language was French, - # but then it was changed to English before checkdbs ran. - text = NL.join(upending) - charset = Charset(mlist.preferred_language.charset) - incodec = charset.input_codec or 'ascii' - outcodec = charset.output_codec or 'ascii' - if isinstance(text, unicode): - return text.encode(outcodec, 'replace') - # Be sure this is a byte string encodeable in the list's charset - utext = unicode(text, incodec, 'replace') - return utext.encode(outcodec, 'replace') - - - -def auto_discard(mlist): - # Discard old held messages - discard_count = 0 - expire = config.days(mlist.max_days_to_hold) - requestsdb = IListRequests(mlist) - heldmsgs = list(requestsdb.of_type(RequestType.held_message)) - if expire and heldmsgs: - for request in heldmsgs: - key, data = requestsdb.get_request(request.id) - if now - data['date'] > expire: - handle_request(mlist, request.id, config.DISCARD) - discard_count += 1 - mlist.Save() - return discard_count - - - -# Figure out epoch seconds of midnight at the start of today (or the given -# 3-tuple date of (year, month, day). -def midnight(date=None): - if date is None: - date = time.localtime()[:3] - # -1 for dst flag tells the library to figure it out - return time.mktime(date + (0,)*5 + (-1,)) - - -def main(): - opts, args, parser = parseargs() - initialize(opts.config) - - for name in config.list_manager.names: - # The list must be locked in order to open the requests database - mlist = MailList.MailList(name) - try: - count = IListRequests(mlist).count - # While we're at it, let's evict yesterday's autoresponse data - midnight_today = midnight() - evictions = [] - for sender in mlist.hold_and_cmd_autoresponses.keys(): - date, respcount = mlist.hold_and_cmd_autoresponses[sender] - if midnight(date) < midnight_today: - evictions.append(sender) - if evictions: - for sender in evictions: - del mlist.hold_and_cmd_autoresponses[sender] - # This is the only place we've changed the list's database - mlist.Save() - if count: - # Set the default language the the list's preferred language. - _.default = mlist.preferred_language - realname = mlist.real_name - discarded = auto_discard(mlist) - if discarded: - count = count - discarded - text = _('Notice: $discarded old request(s) ' - 'automatically expired.\n\n') - else: - text = '' - if count: - text += Utils.maketext( - 'checkdbs.txt', - {'count' : count, - 'mail_host': mlist.mail_host, - 'adminDB' : mlist.GetScriptURL('admindb', - absolute=1), - 'real_name': realname, - }, mlist=mlist) - text += '\n' + pending_requests(mlist) - subject = _('$count $realname moderator ' - 'request(s) waiting') - else: - subject = _('$realname moderator request check result') - msg = UserNotification(mlist.GetOwnerEmail(), - mlist.GetBouncesEmail(), - subject, text, - mlist.preferred_language) - msg.send(mlist, **{'tomoderators': True}) - finally: - mlist.Unlock() - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/config_list.py b/src/mailman/bin/config_list.py deleted file mode 100644 index a0b2a54f4..000000000 --- a/src/mailman/bin/config_list.py +++ /dev/null @@ -1,332 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import re -import sys -import time -import optparse - -from mailman import MailList -from mailman import errors -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.utilities.string import wrap -from mailman.version import MAILMAN_VERSION - - -NL = '\n' -nonasciipat = re.compile(r'[\x80-\xff]') - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] listname - -Configure a list from a text file description, or dump a list's configuration -settings.""")) - parser.add_option('-i', '--inputfile', - metavar='FILENAME', default=None, type='string', - help=_("""\ -Configure the list by assigning each module-global variable in the file to an -attribute on the mailing list object, then save the list. The named file is -loaded with execfile() and must be legal Python code. Any variable that isn't -already an attribute of the list object is ignored (a warning message is -printed). See also the -c option. - -A special variable named 'mlist' is put into the globals during the execfile, -which is bound to the actual MailList object. This lets you do all manner of -bizarre thing to the list object, but BEWARE! Using this can severely (and -possibly irreparably) damage your mailing list! - -The may not be used with the -o option.""")) - parser.add_option('-o', '--outputfile', - metavar='FILENAME', default=None, type='string', - help=_("""\ -Instead of configuring the list, print out a mailing list's configuration -variables in a format suitable for input using this script. In this way, you -can easily capture the configuration settings for a particular list and -imprint those settings on another list. FILENAME is the file to output the -settings to. If FILENAME is `-', standard out is used. - -This may not be used with the -i option.""")) - parser.add_option('-c', '--checkonly', - default=False, action='store_true', help=_("""\ -With this option, the modified list is not actually changed. This is only -useful with the -i option.""")) - parser.add_option('-v', '--verbose', - default=False, action='store_true', help=_("""\ -Print the name of each attribute as it is being changed. This is only useful -with the -i option.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if len(args) > 1: - parser.print_help() - parser.error(_('Unexpected arguments')) - if not args: - parser.error(_('List name is required')) - return parser, opts, args - - - -def do_output(listname, outfile, parser): - closep = False - try: - if outfile == '-': - outfp = sys.stdout - else: - outfp = open(outfile, 'w') - closep = True - # Open the specified list unlocked, since we're only reading it. - try: - mlist = MailList.MailList(listname, lock=False) - except errors.MMListError: - parser.error(_('No such list: $listname')) - # Preamble for the config info. PEP 263 charset and capture time. - charset = mlist.preferred_language.charset - # Set the system's default language. - _.default = mlist.preferred_language.code - if not charset: - charset = 'us-ascii' - when = time.ctime(time.time()) - print >> outfp, _('''\ -# -*- python -*- -# -*- coding: $charset -*- -## "$listname" mailing list configuration settings -## captured on $when -''') - # Get all the list config info. All this stuff is accessible via the - # web interface. - for k in config.ADMIN_CATEGORIES: - subcats = mlist.GetConfigSubCategories(k) - if subcats is None: - do_list_categories(mlist, k, None, outfp) - else: - for subcat in [t[0] for t in subcats]: - do_list_categories(mlist, k, subcat, outfp) - finally: - if closep: - outfp.close() - - - -def do_list_categories(mlist, k, subcat, outfp): - info = mlist.GetConfigInfo(k, subcat) - label, gui = mlist.GetConfigCategories()[k] - if info is None: - return - charset = mlist.preferred_language.charset - print >> outfp, '##', k.capitalize(), _('options') - print >> outfp, '#' - # First, massage the descripton text, which could have obnoxious - # leading whitespace on second and subsequent lines due to - # triple-quoted string nonsense in the source code. - desc = NL.join([s.lstrip() for s in info[0].splitlines()]) - # Print out the category description - desc = wrap(desc) - for line in desc.splitlines(): - print >> outfp, '#', line - print >> outfp - for data in info[1:]: - if not isinstance(data, tuple): - continue - varname = data[0] - # Variable could be volatile - if varname[0] == '_': - continue - vtype = data[1] - # First, massage the descripton text, which could have - # obnoxious leading whitespace on second and subsequent lines - # due to triple-quoted string nonsense in the source code. - desc = NL.join([s.lstrip() for s in data[-1].splitlines()]) - # Now strip out all HTML tags - desc = re.sub('<.*?>', '', desc) - # And convert </> to <> - desc = re.sub('<', '<', desc) - desc = re.sub('>', '>', desc) - # Print out the variable description. - desc = wrap(desc) - for line in desc.split('\n'): - print >> outfp, '#', line - # munge the value based on its type - value = None - if hasattr(gui, 'getValue'): - value = gui.getValue(mlist, vtype, varname, data[2]) - if value is None and not varname.startswith('_'): - value = getattr(mlist, varname) - if vtype in (config.String, config.Text, config.FileUpload): - print >> outfp, varname, '=', - lines = value.splitlines() - if not lines: - print >> outfp, "''" - elif len(lines) == 1: - if charset != 'us-ascii' and nonasciipat.search(lines[0]): - # This is more readable for non-english list. - print >> outfp, '"' + lines[0].replace('"', '\\"') + '"' - else: - print >> outfp, repr(lines[0]) - else: - if charset == 'us-ascii' and nonasciipat.search(value): - # Normally, an english list should not have non-ascii char. - print >> outfp, repr(NL.join(lines)) - else: - outfp.write(' """') - outfp.write(NL.join(lines).replace('"', '\\"')) - outfp.write('"""\n') - elif vtype in (config.Radio, config.Toggle): - print >> outfp, '#' - print >> outfp, '#', _('legal values are:') - # TBD: This is disgusting, but it's special cased - # everywhere else anyway... - if varname == 'subscribe_policy' and \ - not config.ALLOW_OPEN_SUBSCRIBE: - i = 1 - else: - i = 0 - for choice in data[2]: - print >> outfp, '# ', i, '= "%s"' % choice - i += 1 - print >> outfp, varname, '=', repr(value) - else: - print >> outfp, varname, '=', repr(value) - print >> outfp - - - -def getPropertyMap(mlist): - guibyprop = {} - categories = mlist.GetConfigCategories() - for category, (label, gui) in categories.items(): - if not hasattr(gui, 'GetConfigInfo'): - continue - subcats = mlist.GetConfigSubCategories(category) - if subcats is None: - subcats = [(None, None)] - for subcat, sclabel in subcats: - for element in gui.GetConfigInfo(mlist, category, subcat): - if not isinstance(element, tuple): - continue - propname = element[0] - wtype = element[1] - guibyprop[propname] = (gui, wtype) - return guibyprop - - -class FakeDoc: - # Fake the error reporting API for the htmlformat.Document class - def addError(self, s, tag=None, *args): - if tag: - print >> sys.stderr, tag - print >> sys.stderr, s % args - - def set_language(self, val): - pass - - - -def do_input(listname, infile, checkonly, verbose, parser): - fakedoc = FakeDoc() - # Open the specified list locked, unless checkonly is set - try: - mlist = MailList.MailList(listname, lock=not checkonly) - except errors.MMListError as error: - parser.error(_('No such list "$listname"\n$error')) - savelist = False - guibyprop = getPropertyMap(mlist) - try: - globals = {'mlist': mlist} - # Any exception that occurs in execfile() will cause the list to not - # be saved, but any other problems are not save-fatal. - execfile(infile, globals) - savelist = True - for k, v in globals.items(): - if k in ('mlist', '__builtins__'): - continue - if not hasattr(mlist, k): - print >> sys.stderr, _('attribute "$k" ignored') - continue - if verbose: - print >> sys.stderr, _('attribute "$k" changed') - missing = [] - gui, wtype = guibyprop.get(k, (missing, missing)) - if gui is missing: - # This isn't an official property of the list, but that's - # okay, we'll just restore it the old fashioned way - print >> sys.stderr, _('Non-standard property restored: $k') - setattr(mlist, k, v) - else: - # BAW: This uses non-public methods. This logic taken from - # the guts of GUIBase.handleForm(). - try: - validval = gui._getValidValue(mlist, k, wtype, v) - except ValueError: - print >> sys.stderr, _('Invalid value for property: $k') - except errors.EmailAddressError: - print >> sys.stderr, _( - 'Bad email address for option $k: $v') - else: - # BAW: Horrible hack, but then this is special cased - # everywhere anyway. :( Privacy._setValue() knows that - # when ALLOW_OPEN_SUBSCRIBE is false, the web values are - # 0, 1, 2 but these really should be 1, 2, 3, so it adds - # one. But we really do provide [0..3] so we need to undo - # the hack that _setValue adds. :( :( - if k == 'subscribe_policy' and \ - not config.ALLOW_OPEN_SUBSCRIBE: - validval -= 1 - # BAW: Another horrible hack. This one is just too hard - # to fix in a principled way in Mailman 2.1 - elif k == 'new_member_options': - # Because this is a Checkbox, _getValidValue() - # transforms the value into a list of one item. - validval = validval[0] - validval = [bitfield for bitfield, bitval - in config.OPTINFO.items() - if validval & bitval] - gui._setValue(mlist, k, validval, fakedoc) - # BAW: when to do gui._postValidate()??? - finally: - if savelist and not checkonly: - mlist.Save() - mlist.Unlock() - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - listname = args[0] - - # Sanity check - if opts.inputfile and opts.outputfile: - parser.error(_('Only one of -i or -o is allowed')) - if not opts.inputfile and not opts.outputfile: - parser.error(_('One of -i or -o is required')) - - if opts.outputfile: - do_output(listname, opts.outputfile, parser) - else: - do_input(listname, opts.inputfile, opts.checkonly, - opts.verbose, parser) - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/disabled.py b/src/mailman/bin/disabled.py deleted file mode 100644 index b190556c2..000000000 --- a/src/mailman/bin/disabled.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright (C) 2001-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import time -import logging -import optparse - -from mailman import MailList -from mailman import MemberAdaptor -from mailman import Pending -from mailman import loginit -from mailman.Bouncer import _BounceInfo -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.interfaces.member import NotAMemberError -from mailman.version import MAILMAN_VERSION - - -# Work around known problems with some RedHat cron daemons -import signal -signal.signal(signal.SIGCHLD, signal.SIG_DFL) - -ALL = (MemberAdaptor.BYBOUNCE, - MemberAdaptor.BYADMIN, - MemberAdaptor.BYUSER, - MemberAdaptor.UNKNOWN, - ) - - - -def who_callback(option, opt, value, parser): - dest = getattr(parser.values, option.dest) - if opt in ('-o', '--byadmin'): - dest.add(MemberAdaptor.BYADMIN) - elif opt in ('-m', '--byuser'): - dest.add(MemberAdaptor.BYUSER) - elif opt in ('-u', '--unknown'): - dest.add(MemberAdaptor.UNKNOWN) - elif opt in ('-b', '--notbybounce'): - dest.discard(MemberAdaptor.BYBOUNCE) - elif opt in ('-a', '--all'): - dest.update(ALL) -x5o - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Process disabled members, recommended once per day. - -This script iterates through every mailing list looking for members whose -delivery is disabled. If they have been disabled due to bounces, they will -receive another notification, or they may be removed if they've received the -maximum number of notifications. - -Use the --byadmin, --byuser, and --unknown flags to also send notifications to -members whose accounts have been disabled for those reasons. Use --all to -send the notification to all disabled members.""")) - # This is the set of working flags for who to send notifications to. By - # default, we notify anybody who has been disable due to bounces. - parser.set_defaults(who=set([MemberAdaptor.BYBOUNCE])) - parser.add_option('-o', '--byadmin', - callback=who_callback, action='callback', dest='who', - help=_("""\ -Also send notifications to any member disabled by the list -owner/administrator.""")) - parser.add_option('-m', '--byuser', - callback=who_callback, action='callback', dest='who', - help=_("""\ -Also send notifications to any member who has disabled themself.""")) - parser.add_option('-u', '--unknown', - callback=who_callback, action='callback', dest='who', - help=_("""\ -Also send notifications to any member disabled for unknown reasons -(usually a legacy disabled address).""")) - parser.add_option('-b', '--notbybounce', - callback=who_callback, action='callback', dest='who', - help=_("""\ -Don't send notifications to members disabled because of bounces (the -default is to notify bounce disabled members).""")) - parser.add_option('-a', '--all', - callback=who_callback, action='callback', dest='who', - help=_('Send notifications to all disabled members')) - parser.add_option('-f', '--force', - default=False, action='store_true', - help=_("""\ -Send notifications to disabled members even if they're not due a new -notification yet.""")) - parser.add_option('-l', '--listname', - dest='listnames', action='append', default=[], - type='string', help=_("""\ -Process only the given list, otherwise do all lists.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - return opts, args, parser - - - -def main(): - opts, args, parser = parseargs() - config.load(opts.config) - - loginit.initialize(propagate=True) - elog = logging.getLogger('mailman.error') - blog = logging.getLogger('mailman.bounce') - - listnames = set(opts.listnames or config.list_manager.names) - who = tuple(opts.who) - - msg = _('[disabled by periodic sweep and cull, no message available]') - today = time.mktime(time.localtime()[:3] + (0,) * 6) - for listname in listnames: - # List of members to notify - notify = [] - mlist = MailList.MailList(listname) - try: - interval = mlist.bounce_you_are_disabled_warnings_interval - # Find all the members who are currently bouncing and see if - # they've reached the disable threshold but haven't yet been - # disabled. This is a sweep through the membership catching - # situations where they've bounced a bunch, then the list admin - # lowered the threshold, but we haven't (yet) seen more bounces - # from the member. Note: we won't worry about stale information - # or anything else since the normal bounce processing code will - # handle that. - disables = [] - for member in mlist.getBouncingMembers(): - if mlist.getDeliveryStatus(member) <> MemberAdaptor.ENABLED: - continue - info = mlist.getBounceInfo(member) - if info.score >= mlist.bounce_score_threshold: - disables.append((member, info)) - if disables: - for member, info in disables: - mlist.disableBouncingMember(member, info, msg) - # Go through all the members who have delivery disabled, and find - # those that are due to have another notification. If they are - # disabled for another reason than bouncing, and we're processing - # them (because of the command line switch) then they won't have a - # bounce info record. We can piggyback on that for all disable - # purposes. - members = mlist.getDeliveryStatusMembers(who) - for member in members: - info = mlist.getBounceInfo(member) - if not info: - # See if they are bounce disabled, or disabled for some - # other reason. - status = mlist.getDeliveryStatus(member) - if status == MemberAdaptor.BYBOUNCE: - elog.error( - '%s disabled BYBOUNCE lacks bounce info, list: %s', - member, mlist.internal_name()) - continue - info = _BounceInfo( - member, 0, today, - mlist.bounce_you_are_disabled_warnings, - mlist.pend_new(Pending.RE_ENABLE, - mlist.internal_name(), - member)) - mlist.setBounceInfo(member, info) - lastnotice = time.mktime(info.lastnotice + (0,) * 6) - if opts.force or today >= lastnotice + interval: - notify.append(member) - # Now, send notifications to anyone who is due - for member in notify: - blog.info('Notifying disabled member %s for list: %s', - member, mlist.internal_name()) - try: - mlist.sendNextNotification(member) - except NotAMemberError: - # There must have been some problem with the data we have - # on this member. Most likely it's that they don't have a - # password assigned. Log this and delete the member. - blog.info( - 'Cannot send disable notice to non-member: %s', - member) - mlist.ApprovedDeleteMember(member, 'cron/disabled') - mlist.Save() - finally: - mlist.Unlock() - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/docs/master.rst b/src/mailman/bin/docs/master.rst index 3d10b69ac..5a3a94da6 100644 --- a/src/mailman/bin/docs/master.rst +++ b/src/mailman/bin/docs/master.rst @@ -4,10 +4,10 @@ Mailman runner control Mailman has a number of *runner subprocesses* which perform long-running tasks such as listening on an LMTP port, processing REST API requests, or processing -messages in a queue directory. In normal operation, the ``bin/mailman`` -command is used to start, stop and manage the runners. This is just a wrapper -around the real master watcher, which handles runner starting, stopping, -exiting, and log file reopening. +messages in a queue directory. In normal operation, the ``mailman`` command +is used to start, stop and manage the runners. This is just a wrapper around +the real master watcher, which handles runner starting, stopping, exiting, and +log file reopening. >>> from mailman.testing.helpers import TestableMaster diff --git a/src/mailman/bin/export.py b/src/mailman/bin/export.py deleted file mode 100644 index 279abc36f..000000000 --- a/src/mailman/bin/export.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright (C) 2006-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Export an XML representation of a mailing list.""" - -import sys -import codecs -import datetime -import optparse - -from xml.sax.saxutils import escape - -from mailman import Defaults -from mailman import errors -from mailman import MemberAdaptor -from mailman.MailList import MailList -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.version import MAILMAN_VERSION - - -SPACE = ' ' - -TYPES = { - Defaults.Toggle : 'bool', - Defaults.Radio : 'radio', - Defaults.String : 'string', - Defaults.Text : 'text', - Defaults.Email : 'email', - Defaults.EmailList : 'email_list', - Defaults.Host : 'host', - Defaults.Number : 'number', - Defaults.FileUpload : 'upload', - Defaults.Select : 'select', - Defaults.Topics : 'topics', - Defaults.Checkbox : 'checkbox', - Defaults.EmailListEx : 'email_list_ex', - Defaults.HeaderFilter : 'header_filter', - } - - - -class Indenter: - def __init__(self, fp, indentwidth=4): - self._fp = fp - self._indent = 0 - self._width = indentwidth - - def indent(self): - self._indent += 1 - - def dedent(self): - self._indent -= 1 - assert self._indent >= 0 - - def write(self, s): - if s <> '\n': - self._fp.write(self._indent * self._width * ' ') - self._fp.write(s) - - - -class XMLDumper(object): - def __init__(self, fp): - self._fp = Indenter(fp) - self._tagbuffer = None - self._stack = [] - - def _makeattrs(self, tagattrs): - # The attribute values might contain angle brackets. They might also - # be None. - attrs = [] - for k, v in tagattrs.items(): - if v is None: - v = '' - else: - v = escape(str(v)) - attrs.append('%s="%s"' % (k, v)) - return SPACE.join(attrs) - - def _flush(self, more=True): - if not self._tagbuffer: - return - name, attributes = self._tagbuffer - self._tagbuffer = None - if attributes: - attrstr = ' ' + self._makeattrs(attributes) - else: - attrstr = '' - if more: - print >> self._fp, '<%s%s>' % (name, attrstr) - self._fp.indent() - self._stack.append(name) - else: - print >> self._fp, '<%s%s/>' % (name, attrstr) - - # Use this method when you know you have sub-elements. - def _push_element(self, _name, **_tagattrs): - self._flush() - self._tagbuffer = (_name, _tagattrs) - - def _pop_element(self, _name): - buffered = bool(self._tagbuffer) - self._flush(more=False) - if not buffered: - name = self._stack.pop() - assert name == _name, 'got: %s, expected: %s' % (_name, name) - self._fp.dedent() - print >> self._fp, '</%s>' % name - - # Use this method when you do not have sub-elements - def _element(self, _name, _value=None, **_attributes): - self._flush() - if _attributes: - attrs = ' ' + self._makeattrs(_attributes) - else: - attrs = '' - if _value is None: - print >> self._fp, '<%s%s/>' % (_name, attrs) - else: - # The value might contain angle brackets. - value = escape(_value.decode('utf-8')) - print >> self._fp, '<%s%s>%s</%s>' % (_name, attrs, value, _name) - - def _do_list_categories(self, mlist, k, subcat=None): - info = mlist.GetConfigInfo(k, subcat) - label, gui = mlist.GetConfigCategories()[k] - if info is None: - return - for data in info[1:]: - if not isinstance(data, tuple): - continue - varname = data[0] - # Variable could be volatile - if varname.startswith('_'): - continue - vtype = data[1] - # Munge the value based on its type - value = None - if hasattr(gui, 'getValue'): - value = gui.getValue(mlist, vtype, varname, data[2]) - if value is None: - value = getattr(mlist, varname) - widget_type = TYPES[vtype] - if isinstance(value, list): - self._push_element('option', name=varname, type=widget_type) - for v in value: - self._element('value', v) - self._pop_element('option') - else: - self._element('option', value, name=varname, type=widget_type) - - def _dump_list(self, mlist): - # Write list configuration values - self._push_element('list', name=mlist.fqdn_listname) - self._push_element('configuration') - self._element('option', - mlist.preferred_language, - name='preferred_language') - for k in config.ADMIN_CATEGORIES: - subcats = mlist.GetConfigSubCategories(k) - if subcats is None: - self._do_list_categories(mlist, k) - else: - for subcat in [t[0] for t in subcats]: - self._do_list_categories(mlist, k, subcat) - self._pop_element('configuration') - # Write membership - self._push_element('roster') - digesters = set(mlist.getDigestMemberKeys()) - for member in sorted(mlist.getMembers()): - attrs = dict(id=member) - cased = mlist.getMemberCPAddress(member) - if cased <> member: - attrs['original'] = cased - self._push_element('member', **attrs) - self._element('realname', mlist.getMemberName(member)) - self._element('password', mlist.getMemberPassword(member)) - self._element('language', mlist.getMemberLanguage(member)) - # Delivery status, combined with the type of delivery - attrs = {} - status = mlist.getDeliveryStatus(member) - if status == MemberAdaptor.ENABLED: - attrs['status'] = 'enabled' - else: - attrs['status'] = 'disabled' - attrs['reason'] = {MemberAdaptor.BYUSER : 'byuser', - MemberAdaptor.BYADMIN : 'byadmin', - MemberAdaptor.BYBOUNCE : 'bybounce', - }.get(mlist.getDeliveryStatus(member), - 'unknown') - if member in digesters: - if mlist.getMemberOption(member, Defaults.DisableMime): - attrs['delivery'] = 'plain' - else: - attrs['delivery'] = 'mime' - else: - attrs['delivery'] = 'regular' - changed = mlist.getDeliveryStatusChangeTime(member) - if changed: - when = datetime.datetime.fromtimestamp(changed) - attrs['changed'] = when.isoformat() - self._element('delivery', **attrs) - for option, flag in Defaults.OPTINFO.items(): - # Digest/Regular delivery flag must be handled separately - if option in ('digest', 'plain'): - continue - value = mlist.getMemberOption(member, flag) - self._element(option, value) - topics = mlist.getMemberTopics(member) - if not topics: - self._element('topics') - else: - self._push_element('topics') - for topic in topics: - self._element('topic', topic) - self._pop_element('topics') - self._pop_element('member') - self._pop_element('roster') - self._pop_element('list') - - def dump(self, listnames): - print >> self._fp, '<?xml version="1.0" encoding="UTF-8"?>' - self._push_element('mailman', **{ - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:noNamespaceSchemaLocation': 'ssi-1.0.xsd', - }) - for listname in sorted(listnames): - try: - mlist = MailList(listname, lock=False) - except errors.MMUnknownListError: - print >> sys.stderr, _('No such list: $listname') - continue - self._dump_list(mlist) - self._pop_element('mailman') - - def close(self): - while self._stack: - self._pop_element() - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Export the configuration and members of a mailing list in XML format.""")) - parser.add_option('-o', '--outputfile', - metavar='FILENAME', default=None, type='string', - help=_("""\ -Output XML to FILENAME. If not given, or if FILENAME is '-', standard out is -used.""")) - parser.add_option('-l', '--listname', - default=[], action='append', type='string', - metavar='LISTNAME', dest='listnames', help=_("""\ -The list to include in the output. If not given, then all mailing lists are -included in the XML output. Multiple -l flags may be given.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - parser.error(_('Unexpected arguments')) - return parser, opts, args - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - close = False - if opts.outputfile in (None, '-'): - writer = codecs.getwriter('utf-8') - fp = writer(sys.stdout) - else: - fp = codecs.open(opts.outputfile, 'w', 'utf-8') - close = True - - try: - dumper = XMLDumper(fp) - if opts.listnames: - listnames = [] - for listname in opts.listnames: - if '@' not in listname: - listname = '%s@%s' % (listname, config.DEFAULT_EMAIL_HOST) - listnames.append(listname) - else: - listnames = config.list_manager.names - dumper.dump(listnames) - dumper.close() - finally: - if close: - fp.close() diff --git a/src/mailman/bin/find_member.py b/src/mailman/bin/find_member.py deleted file mode 100644 index 349af8247..000000000 --- a/src/mailman/bin/find_member.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import re -import sys -import optparse - -from mailman import errors -from mailman import MailList -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - - -AS_MEMBER = 0x01 -AS_OWNER = 0x02 - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] regex [regex ...] - -Find all lists that a member's address is on. - -The interaction between -l and -x (see below) is as follows. If any -l option -is given then only the named list will be included in the search. If any -x -option is given but no -l option is given, then all lists will be search -except those specifically excluded. - -Regular expression syntax uses the Python 're' module. Complete -specifications are at: - -http://www.python.org/doc/current/lib/module-re.html - -Address matches are case-insensitive, but case-preserved addresses are -displayed.""")) - parser.add_option('-l', '--listname', - type='string', default=[], action='append', - dest='listnames', - help=_('Include only the named list in the search')) - parser.add_option('-x', '--exclude', - type='string', default=[], action='append', - dest='excludes', - help=_('Exclude the named list from the search')) - parser.add_option('-w', '--owners', - default=False, action='store_true', - help=_('Search list owners as well as members')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if not args: - parser.print_help() - print >> sys.stderr, _('Search regular expression required') - sys.exit(1) - return parser, opts, args - - - -def main(): - parser, opts, args = parseargs() - config.load(opts.config) - - listnames = opts.listnames or config.list_manager.names - includes = set(listname.lower() for listname in listnames) - excludes = set(listname.lower() for listname in opts.excludes) - listnames = includes - excludes - - if not listnames: - print _('No lists to search') - return - - cres = [] - for r in args: - cres.append(re.compile(r, re.IGNORECASE)) - # dictionary of {address, (listname, ownerp)} - matches = {} - for listname in listnames: - try: - mlist = MailList.MailList(listname, lock=False) - except errors.MMListError: - print _('No such list: $listname') - continue - if opts.owners: - owners = mlist.owner - else: - owners = [] - for cre in cres: - for member in mlist.getMembers(): - if cre.search(member): - addr = mlist.getMemberCPAddress(member) - entries = matches.get(addr, {}) - aswhat = entries.get(listname, 0) - aswhat |= AS_MEMBER - entries[listname] = aswhat - matches[addr] = entries - for owner in owners: - if cre.search(owner): - entries = matches.get(owner, {}) - aswhat = entries.get(listname, 0) - aswhat |= AS_OWNER - entries[listname] = aswhat - matches[owner] = entries - addrs = matches.keys() - addrs.sort() - for k in addrs: - hits = matches[k] - lists = hits.keys() - print k, _('found in:') - for name in lists: - aswhat = hits[name] - if aswhat & AS_MEMBER: - print ' ', name - if aswhat & AS_OWNER: - print ' ', name, _('(as owner)') - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/gate_news.py b/src/mailman/bin/gate_news.py deleted file mode 100644 index 72568cd1b..000000000 --- a/src/mailman/bin/gate_news.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys -import time -import socket -import logging -import nntplib -import optparse -import email.Errors - -from email.Parser import Parser -from flufl.lock import Lock, TimeOutError -from lazr.config import as_host_port - -from mailman import MailList -from mailman import Message -from mailman import loginit -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.core.switchboard import Switchboard -from mailman.version import MAILMAN_VERSION - -# Work around known problems with some RedHat cron daemons -import signal -signal.signal(signal.SIGCHLD, signal.SIG_DFL) - -NL = '\n' - -log = None - -class _ContinueLoop(Exception): - pass - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Poll the NNTP servers for messages to be gatewayed to mailing lists.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return opts, args, parser - - - -_hostcache = {} - -def open_newsgroup(mlist): - # Split host:port if given. - nntp_host, nntp_port = as_host_port(mlist.nntp_host, default_port=119) - # Open up a "mode reader" connection to nntp server. This will be shared - # for all the gated lists having the same nntp_host. - conn = _hostcache.get(mlist.nntp_host) - if conn is None: - try: - conn = nntplib.NNTP(nntp_host, nntp_port, - readermode=True, - user=config.NNTP_USERNAME, - password=config.NNTP_PASSWORD) - except (socket.error, nntplib.NNTPError, IOError) as e: - log.error('error opening connection to nntp_host: %s\n%s', - mlist.nntp_host, e) - raise - _hostcache[mlist.nntp_host] = conn - # Get the GROUP information for the list, but we're only really interested - # in the first article number and the last article number - r, c, f, l, n = conn.group(mlist.linked_newsgroup) - return conn, int(f), int(l) - - -def clearcache(): - for conn in set(_hostcache.values()): - conn.quit() - _hostcache.clear() - - - -# This function requires the list to be locked. -def poll_newsgroup(mlist, conn, first, last, glock): - listname = mlist.internal_name() - # NEWNEWS is not portable and has synchronization issues. - for num in range(first, last): - glock.refresh() - try: - headers = conn.head(repr(num))[3] - found_to = False - beenthere = False - for header in headers: - i = header.find(':') - value = header[:i].lower() - if i > 0 and value == 'to': - found_to = True - # FIXME 2010-02-16 barry use List-Post header. - if value <> 'x-beenthere': - continue - if header[i:] == ': %s' % mlist.posting_address: - beenthere = True - break - if not beenthere: - body = conn.body(repr(num))[3] - # Usenet originated messages will not have a Unix envelope - # (i.e. "From " header). This breaks Pipermail archiving, so - # we will synthesize one. Be sure to use the format searched - # for by mailbox.UnixMailbox._isrealfromline(). BAW: We use - # the -bounces address here in case any downstream clients use - # the envelope sender for bounces; I'm not sure about this, - # but it's the closest to the old semantics. - lines = ['From %s %s' % (mlist.GetBouncesEmail(), - time.ctime(time.time()))] - lines.extend(headers) - lines.append('') - lines.extend(body) - lines.append('') - p = Parser(Message.Message) - try: - msg = p.parsestr(NL.join(lines)) - except email.Errors.MessageError as e: - log.error('email package exception for %s:%d\n%s', - mlist.linked_newsgroup, num, e) - raise _ContinueLoop - if found_to: - del msg['X-Originally-To'] - msg['X-Originally-To'] = msg['To'] - del msg['To'] - msg['To'] = mlist.posting_address - # Post the message to the locked list - inq = Switchboard(config.INQUEUE_DIR) - inq.enqueue(msg, - listid=mlist.list_id, - fromusenet=True) - log.info('posted to list %s: %7d', listname, num) - except nntplib.NNTPError as e: - log.exception('NNTP error for list %s: %7d', listname, num) - except _ContinueLoop: - continue - # Even if we don't post the message because it was seen on the - # list already, update the watermark - mlist.usenet_watermark = num - - - -def process_lists(glock): - for listname in config.list_manager.names: - glock.refresh() - # Open the list unlocked just to check to see if it is gating news to - # mail. If not, we're done with the list. Otherwise, lock the list - # and gate the group. - mlist = MailList.MailList(listname, lock=False) - if not mlist.gateway_to_mail: - continue - # Get the list's watermark, i.e. the last article number that we gated - # from news to mail. None means that this list has never polled its - # newsgroup and that we should do a catch up. - watermark = getattr(mlist, 'usenet_watermark', None) - # Open the newsgroup, but let most exceptions percolate up. - try: - conn, first, last = open_newsgroup(mlist) - except (socket.error, nntplib.NNTPError): - break - log.info('%s: [%d..%d]', listname, first, last) - try: - try: - if watermark is None: - mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) - # This is the first time we've tried to gate this - # newsgroup. We essentially do a mass catch-up, otherwise - # we'd flood the mailing list. - mlist.usenet_watermark = last - log.info('%s caught up to article %d', listname, last) - else: - # The list has been polled previously, so now we simply - # grab all the messages on the newsgroup that have not - # been seen by the mailing list. The first such article - # is the maximum of the lowest article available in the - # newsgroup and the watermark. It's possible that some - # articles have been expired since the last time gate_news - # has run. Not much we can do about that. - start = max(watermark + 1, first) - if start > last: - log.info('nothing new for list %s', listname) - else: - mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) - log.info('gating %s articles [%d..%d]', - listname, start, last) - # Use last+1 because poll_newsgroup() employes a for - # loop over range, and this will not include the last - # element in the list. - poll_newsgroup(mlist, conn, start, last + 1, glock) - except TimeOutError: - log.error('Could not acquire list lock: %s', listname) - finally: - if mlist.Locked(): - mlist.Save() - mlist.Unlock() - log.info('%s watermark: %d', listname, mlist.usenet_watermark) - - - -def main(): - opts, args, parser = parseargs() - config.load(opts.config) - - GATENEWS_LOCK_FILE = os.path.join(config.LOCK_DIR, 'gate_news.lock') - LOCK_LIFETIME = config.hours(2) - - loginit.initialize(propagate=True) - log = logging.getLogger('mailman.fromusenet') - - try: - with Lock(GATENEWS_LOCK_FILE, - # It's okay to hijack this - lifetime=LOCK_LIFETIME) as lock: - process_lists(lock) - clearcache() - except TimeOutError: - log.error('Could not acquire gate_news lock') - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/list_owners.py b/src/mailman/bin/list_owners.py deleted file mode 100644 index 5b5fca2bf..000000000 --- a/src/mailman/bin/list_owners.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (C) 2002-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import sys -import optparse - -from zope.component import getUtility - -from mailman.MailList import MailList -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.interfaces.listmanager import IListManager -from mailman.version import MAILMAN_VERSION - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] [listname ...] - -List the owners of a mailing list, or all mailing lists if no list names are -given.""")) - parser.add_option('-w', '--with-listnames', - default=False, action='store_true', - help=_("""\ -Group the owners by list names and include the list names in the output. -Otherwise, the owners will be sorted and uniquified based on the email -address.""")) - parser.add_option('-m', '--moderators', - default=False, action='store_true', - help=_('Include the list moderators in the output.')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - return parser, opts, args - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - list_manager = getUtility(IListManager) - listnames = set(args or list_manager.names) - bylist = {} - - for listname in listnames: - mlist = list_manager.get(listname) - addrs = [addr.address for addr in mlist.owners.addresses] - if opts.moderators: - addrs.extend([addr.address for addr in mlist.moderators.addresses]) - bylist[listname] = addrs - - if opts.with_listnames: - for listname in listnames: - unique = set() - for addr in bylist[listname]: - unique.add(addr) - keys = list(unique) - keys.sort() - print listname - for k in keys: - print '\t', k - else: - unique = set() - for listname in listnames: - for addr in bylist[listname]: - unique.add(addr) - for k in sorted(unique): - print k - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py index 8814cdfc4..3865fef19 100644 --- a/src/mailman/bin/mailman.py +++ b/src/mailman/bin/mailman.py @@ -36,7 +36,7 @@ from zope.interface.verify import verifyObject def main(): - """bin/mailman""" + """The `mailman` command dispatcher.""" # Create the basic parser and add all globally common options. parser = argparse.ArgumentParser( description=_("""\ diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index 492a6b138..5ffe59647 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -320,7 +320,7 @@ class Loop: log.info('Master watcher caught SIGUSR1. Exiting.') signal.signal(signal.SIGUSR1, sigusr1_handler) # SIGTERM is what init will kill this process with when changing run - # levels. It's also the signal 'bin/mailman stop' uses. + # levels. It's also the signal 'mailman stop' uses. def sigterm_handler(signum, frame): for pid in self._kids: os.kill(pid, signal.SIGTERM) diff --git a/src/mailman/bin/runner.py b/src/mailman/bin/runner.py index e8c68dad9..87d11dbe9 100644 --- a/src/mailman/bin/runner.py +++ b/src/mailman/bin/runner.py @@ -108,19 +108,18 @@ def main(): description=_("""\ Start a runner - The runner named on the command line is started, and it can - either run through its main loop once (for those runners that - support this) or continuously. The latter is how the master - runner starts all its subprocesses. + The runner named on the command line is started, and it can either run + through its main loop once (for those runners that support this) or + continuously. The latter is how the master runner starts all its + subprocesses. - -r is required unless -l or -h is given, and its argument must - be one of the names displayed by the -l switch. + -r is required unless -l or -h is given, and its argument must be one + of the names displayed by the -l switch. - Normally, this script should be started from 'bin/mailman - start'. Running it separately or with -o is generally useful - only for debugging. When run this way, the environment variable - $MAILMAN_UNDER_MASTER_CONTROL will be set which subtly changes - some error handling behavior. + Normally, this script should be started from 'mailman start'. Running + it separately or with -o is generally useful only for debugging. When + run this way, the environment variable $MAILMAN_UNDER_MASTER_CONTROL + will be set which subtly changes some error handling behavior. """)) parser.add_argument( '--version', diff --git a/src/mailman/bin/senddigests.py b/src/mailman/bin/senddigests.py deleted file mode 100644 index 59c03de2d..000000000 --- a/src/mailman/bin/senddigests.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys -import optparse - -from mailman import MailList -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.version import MAILMAN_VERSION - -# Work around known problems with some RedHat cron daemons -import signal -signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Dispatch digests for lists w/pending messages and digest_send_periodic -set.""")) - parser.add_option('-l', '--listname', - type='string', default=[], action='append', - dest='listnames', help=_("""\ -Send the digest for the given list only, otherwise the digests for all -lists are sent out. Multiple -l options may be given.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return opts, args, parser - - - -def main(): - opts, args, parser = parseargs() - initialize(opts.config) - - for listname in set(opts.listnames or config.list_manager.names): - mlist = MailList.MailList(listname, lock=False) - if mlist.digest_send_periodic: - mlist.Lock() - try: - try: - mlist.send_digest_now() - mlist.Save() - # We are unable to predict what exception may occur in digest - # processing and we don't want to lose the other digests, so - # we catch everything. - except Exception as errmsg: - print >> sys.stderr, \ - 'List: %s: problem processing %s:\n%s' % \ - (listname, - os.path.join(mlist.data_path, 'digest.mbox'), - errmsg) - finally: - mlist.Unlock() - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/show_config.py b/src/mailman/bin/show_config.py deleted file mode 100644 index 290840ae3..000000000 --- a/src/mailman/bin/show_config.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (C) 2006-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import re -import sys -import pprint -import optparse - -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - - -# List of names never to show even if --verbose -NEVER_SHOW = ['__builtins__', '__doc__'] - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%%prog [options] [pattern ...] - -Show the values of various Defaults.py/mailman.cfg variables. -If one or more patterns are given, show only those variables -whose names match a pattern""")) - parser.add_option('-v', '--verbose', - default=False, action='store_true', - help=_( -"Show all configuration names, not just 'settings'.")) - parser.add_option('-i', '--ignorecase', - default=False, action='store_true', - help=_("Match patterns case-insensitively.")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - return parser, opts, args - - - -def main(): - parser, opts, args = parseargs() - - patterns = [] - if opts.ignorecase: - flag = re.IGNORECASE - else: - flag = 0 - for pattern in args: - patterns.append(re.compile(pattern, flag)) - - pp = pprint.PrettyPrinter(indent=4) - config.load(opts.config) - names = config.__dict__.keys() - names.sort() - for name in names: - if name in NEVER_SHOW: - continue - if not opts.verbose: - if name.startswith('_') or re.search('[a-z]', name): - continue - if patterns: - hit = False - for pattern in patterns: - if pattern.search(name): - hit = True - break - if not hit: - continue - value = config.__dict__[name] - if isinstance(value, str): - if re.search('\n', value): - print '%s = """%s"""' %(name, value) - else: - print "%s = '%s'" % (name, value) - else: - print '%s = ' % name, - pp.pprint(value) - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/commands/cli_inject.py b/src/mailman/commands/cli_inject.py index c467c2508..1b7f15f7b 100644 --- a/src/mailman/commands/cli_inject.py +++ b/src/mailman/commands/cli_inject.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""bin/mailman inject""" +"""The `mailman inject` subcommand.""" __all__ = [ 'Inject', diff --git a/src/mailman/commands/cli_lists.py b/src/mailman/commands/cli_lists.py index 05aa7b7ca..3d5fcd634 100644 --- a/src/mailman/commands/cli_lists.py +++ b/src/mailman/commands/cli_lists.py @@ -86,7 +86,7 @@ class Lists: mlist = list_manager.get(fqdn_name) if args.advertised and not mlist.advertised: continue - domains = getattr(args, 'domains', None) + domains = getattr(args, 'domain', None) if domains and mlist.mail_host not in domains: continue mailing_lists.append(mlist) diff --git a/src/mailman/commands/cli_status.py b/src/mailman/commands/cli_status.py index 34420954b..7faab7941 100644 --- a/src/mailman/commands/cli_status.py +++ b/src/mailman/commands/cli_status.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""bin/mailman status.""" +"""The `mailman status` subcommand.""" __all__ = [ 'Status', diff --git a/src/mailman/commands/cli_withlist.py b/src/mailman/commands/cli_withlist.py index c0c9b3202..e3307d7b4 100644 --- a/src/mailman/commands/cli_withlist.py +++ b/src/mailman/commands/cli_withlist.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""bin/mailman withlist""" +"""The `mailman shell` subcommand.""" __all__ = [ 'Shell', @@ -190,7 +190,7 @@ Programmatically, you can write a function to operate on a mailing list, and this script will take care of the housekeeping (see below for examples). In that case, the general usage syntax is: - % bin/mailman withlist [options] listname [args ...]""")) + % mailman withlist [options] listname [args ...]""")) print() print(_("""\ Here's an example of how to use the --run option. Say you have a file in the @@ -207,7 +207,7 @@ functions: You can print the list's posting address by running the following from the command line: - % bin/mailman withlist -r listaddr mylist@example.com + % mailman withlist -r listaddr mylist@example.com Importing listaddr ... Running listaddr.listaddr() ... mylist@example.com""")) @@ -215,7 +215,7 @@ command line: print(_("""\ And you can print the list's request address by running: - % bin/mailman withlist -r listaddr.requestaddr mylist + % mailman withlist -r listaddr.requestaddr mylist Importing listaddr ... Running listaddr.requestaddr() ... mylist-request@example.com""")) @@ -232,7 +232,7 @@ mailing list. You could put the following function in a file called and run this from the command line: - % bin/mailman withlist -r change mylist@example.com 'My List'""")) + % mailman withlist -r change mylist@example.com 'My List'""")) diff --git a/src/mailman/commands/docs/aliases.rst b/src/mailman/commands/docs/aliases.rst index 528a77770..75a9d3c11 100644 --- a/src/mailman/commands/docs/aliases.rst +++ b/src/mailman/commands/docs/aliases.rst @@ -6,7 +6,7 @@ For some mail servers, Mailman must generate data files that are used to hook Mailman up to the mail server. The details of this differ for each mail server. Generally these files are automatically kept up-to-date when mailing lists are created or removed, but you might occasionally need to manually -regenerate the file. The ``bin/mailman aliases`` command does this. +regenerate the file. The ``mailman aliases`` command does this. >>> class FakeArgs: ... directory = None diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst index 0ff5064bb..1db85918f 100644 --- a/src/mailman/commands/docs/conf.rst +++ b/src/mailman/commands/docs/conf.rst @@ -2,13 +2,13 @@ Display configuration values ============================ -Just like the `Postfix command postconf(1)`_, the ``bin/mailman conf`` command +Just like the `Postfix command postconf(1)`_, the ``mailman conf`` command lets you dump one or more Mailman configuration variables to standard output or a file. Mailman's configuration is divided in multiple sections which contain multiple -key-value pairs. The ``bin/mailman conf`` command allows you to display -a specific key-value pair, or several key-value pairs. +key-value pairs. The ``mailman conf`` command allows you to display a +specific key-value pair, or several key-value pairs. >>> class FakeArgs: ... key = None diff --git a/src/mailman/commands/docs/lists.rst b/src/mailman/commands/docs/lists.rst index 036147a23..04e0d744d 100644 --- a/src/mailman/commands/docs/lists.rst +++ b/src/mailman/commands/docs/lists.rst @@ -100,14 +100,14 @@ You can narrow the search down to a specific domain with the --domain option. A helpful message is displayed if no matching domains are given. >>> FakeArgs.quiet = False - >>> FakeArgs.domains = ['example.org'] + >>> FakeArgs.domain = ['example.org'] >>> command.process(FakeArgs) No matching mailing lists found But if a matching domain is given, only mailing lists in that domain are shown. - >>> FakeArgs.domains = ['example.net'] + >>> FakeArgs.domain = ['example.net'] >>> command.process(FakeArgs) 1 matching mailing lists found: list-one@example.net @@ -115,7 +115,7 @@ shown. More than one --domain argument can be given; then all mailing lists in matching domains are shown. - >>> FakeArgs.domains = ['example.com', 'example.net'] + >>> FakeArgs.domain = ['example.com', 'example.net'] >>> command.process(FakeArgs) 3 matching mailing lists found: list-one@example.com @@ -131,7 +131,7 @@ knowledge. Non-advertised lists are considered private. Display through the command line can select on this attribute. :: - >>> FakeArgs.domains = [] + >>> FakeArgs.domain = [] >>> FakeArgs.advertised = True >>> mlist_1.advertised = False diff --git a/src/mailman/commands/docs/members.rst b/src/mailman/commands/docs/members.rst index 490287235..c90418181 100644 --- a/src/mailman/commands/docs/members.rst +++ b/src/mailman/commands/docs/members.rst @@ -2,8 +2,8 @@ Managing members ================ -The ``bin/mailman members`` command allows a site administrator to display, -add, and remove members from a mailing list. +The ``mailman members`` command allows a site administrator to display, add, +and remove members from a mailing list. :: >>> mlist1 = create_list('test1@example.com') diff --git a/src/mailman/commands/tests/test_create.py b/src/mailman/commands/tests/test_create.py index d9e90df26..d7e17e5d2 100644 --- a/src/mailman/commands/tests/test_create.py +++ b/src/mailman/commands/tests/test_create.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Test `bin/mailman create`.""" +"""Test the `mailman create` subcommand.""" __all__ = [ 'TestCreate', @@ -51,8 +51,6 @@ class FakeParser: class TestCreate(unittest.TestCase): - """Test `bin/mailman create`.""" - layer = ConfigLayer def setUp(self): diff --git a/src/mailman/commands/tests/test_lists.py b/src/mailman/commands/tests/test_lists.py new file mode 100644 index 000000000..dad15eec8 --- /dev/null +++ b/src/mailman/commands/tests/test_lists.py @@ -0,0 +1,67 @@ +# Copyright (C) 2015 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Additional tests for the `lists` command line subcommand.""" + +__all__ = [ + 'TestLists', + ] + + +import unittest + +from io import StringIO +from mailman.app.lifecycle import create_list +from mailman.commands.cli_lists import Lists +from mailman.interfaces.domain import IDomainManager +from mailman.testing.layers import ConfigLayer +from unittest.mock import patch +from zope.component import getUtility + + +class FakeArgs: + advertised = False + names = False + descriptions = False + quiet = False + domain = [] + + +class TestLists(unittest.TestCase): + layer = ConfigLayer + + def test_lists_with_domain_option(self): + # LP: #1166911 - non-matching lists were returned. + getUtility(IDomainManager).add( + 'example.net', 'An example domain.', + 'http://lists.example.net', 'postmaster@example.net') + create_list('test1@example.com') + create_list('test2@example.com') + # Only this one should show up. + create_list('test3@example.net') + create_list('test4@example.com') + command = Lists() + args = FakeArgs() + args.domain.append('example.net') + output = StringIO() + with patch('sys.stdout', output): + command.process(args) + lines = output.getvalue().splitlines() + # The first line is the heading, so skip that. + lines.pop(0) + self.assertEqual(len(lines), 1, lines) + self.assertEqual(lines[0], 'test3@example.net') diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 2b14419a2..d23bdda13 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -149,10 +149,10 @@ class Configuration: # First, collect all variables in a substitution dictionary. $VAR_DIR # is taken from the environment or from the configuration file if the # environment is not set. Because the var_dir setting in the config - # file could be a relative path, and because 'bin/mailman start' - # chdirs to $VAR_DIR, without this subprocesses bin/master and - # bin/runner will create $VAR_DIR hierarchies under $VAR_DIR when that - # path is relative. + # file could be a relative path, and because 'mailman start' chdirs to + # $VAR_DIR, without this subprocesses bin/master and bin/runner will + # create $VAR_DIR hierarchies under $VAR_DIR when that path is + # relative. var_dir = os.environ.get('MAILMAN_VAR_DIR', category.var_dir) substitutions = dict( cwd = os.getcwd(), diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 61b65fac4..f8c3a117e 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -66,10 +66,10 @@ filtered_messages_are_preservable: no [shell] -# `bin/mailman shell` (also `withlist`) gives you an interactive prompt that -# you can use to interact with an initialized and configured Mailman system. -# Use --help for more information. This section allows you to configure -# certain aspects of this interactive shell. +# `mailman shell` (also `withlist`) gives you an interactive prompt that you +# can use to interact with an initialized and configured Mailman system. Use +# --help for more information. This section allows you to configure certain +# aspects of this interactive shell. # Customize the interpreter prompt. prompt: >>> @@ -100,7 +100,7 @@ var_dir: /var/tmp/mailman queue_dir: $var_dir/queue # This is the directory containing the Mailman 'runner' and 'master' commands # if set to the string '$argv', it will be taken as the directory containing -# the 'bin/mailman' command. +# the 'mailman' command. bin_dir: $argv # All list-specific data. list_data_dir: $var_dir/lists diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index 044efea2f..d5b5a6a5f 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -130,8 +130,8 @@ def initialize_1(config_path=None): # PostgreSQL. extra_cfg_path = os.environ.get('MAILMAN_EXTRA_TESTING_CFG') if extra_cfg_path is not None: - with open(extra_cfg_path) as fp: - extra_cfg = fp.read().decode('utf-8') + with open(extra_cfg_path, 'r', encoding='utf-8') as fp: + extra_cfg = fp.read() mailman.config.config.push('extra testing config', extra_cfg) diff --git a/src/mailman/core/logging.py b/src/mailman/core/logging.py index c1db8a55e..7c80037f6 100644 --- a/src/mailman/core/logging.py +++ b/src/mailman/core/logging.py @@ -103,8 +103,8 @@ def _init_logger(propagate, sub_name, log, logger_config): # Get settings from log configuration file (or defaults). log_format = logger_config.format log_datefmt = logger_config.datefmt - # Propagation to the root logger is how we handle logging to stderr - # when the runners are not run as a subprocess of 'bin/mailman start'. + # Propagation to the root logger is how we handle logging to stderr when + # the runners are not run as a subprocess of 'mailman start'. log.propagate = (as_boolean(logger_config.propagate) if propagate is None else propagate) # Set the logger's level. diff --git a/src/mailman/database/transaction.py b/src/mailman/database/transaction.py index 30710017e..b96562d3f 100644 --- a/src/mailman/database/transaction.py +++ b/src/mailman/database/transaction.py @@ -70,6 +70,9 @@ def dbconnection(function): attribute. This calls the function with `store` as the first argument. """ def wrapper(*args, **kws): - # args[0] is self. - return function(args[0], config.db.store, *args[1:], **kws) + # args[0] is self, if there is one. + if len(args) > 0: + return function(args[0], config.db.store, *args[1:], **kws) + else: + return function(config.db.store, **kws) return wrapper diff --git a/src/mailman/docs/8-miles-high.rst b/src/mailman/docs/8-miles-high.rst index 85b186fc5..ae3074e1c 100644 --- a/src/mailman/docs/8-miles-high.rst +++ b/src/mailman/docs/8-miles-high.rst @@ -162,9 +162,9 @@ when the Mailman daemon starts, and what queue the Runner manages. Shell Commands ============== -`bin/mailman`: This is an ubercommand, with subcommands for all the various -things admins might want to do, similar to Mailman 2's mailmanctl, but with -more functionality. +`mailman`: This is an ubercommand, with subcommands for all the various things +admins might want to do, similar to Mailman 2's mailmanctl, but with more +functionality. `bin/master`: The runner manager: starts, watches, stops the runner daemons. diff --git a/src/mailman/docs/DEVELOP.rst b/src/mailman/docs/DEVELOP.rst index f1225658e..5b3ee602a 100644 --- a/src/mailman/docs/DEVELOP.rst +++ b/src/mailman/docs/DEVELOP.rst @@ -72,10 +72,10 @@ queue. You can think of these as fairly typical server process, and examples include the LMTP server, and the HTTP server for processing REST commands. All of the runners are managed by a *master watcher* process. When you type -``bin/mailman start`` you are actually starting the master. Based on +``mailman start`` you are actually starting the master. Based on configuration options, the master will start the appropriate runners as subprocesses, and it will watch for the clean exiting of these subprocesses -when ``bin/mailman stop`` is called. +when ``mailman stop`` is called. Rules and chains diff --git a/src/mailman/docs/MTA.rst b/src/mailman/docs/MTA.rst index 1bc9c6c13..a10c8f3cf 100644 --- a/src/mailman/docs/MTA.rst +++ b/src/mailman/docs/MTA.rst @@ -143,7 +143,7 @@ Transport maps By default, Mailman works well with Postfix transport maps as a way to deliver incoming messages to Mailman's LMTP server. Mailman will automatically write -the correct transport map when its ``bin/mailman aliases`` command is run, or +the correct transport map when its ``mailman aliases`` command is run, or whenever a mailing list is created or removed via other commands. To connect Postfix to Mailman's LMTP server, add the following to Postfix's ``main.cf`` file:: diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 4cc5fc2e3..0c1f6bc63 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -21,6 +21,11 @@ Bugs (LP: #1418280) * When deleting a user via REST, make sure all linked addresses are deleted. Found by Andrew Stuart. (LP: #1419519) + * When trying to subscribe an address to a mailing list through the REST API + where a case-differing version of the address is already subscribed, return + a 409 error instead of a 500 error. Found by Ankush Sharma. (LP: #1425359) + * ``mailman lists --domain`` was not properly handling its arguments. Given + by Manish Gill. (LP: #1166911) Configuration ------------- @@ -44,6 +49,11 @@ REST from the various queue directories via the ``<api>/queues`` resource. * You can now DELETE an address. If the address is linked to a user, the user is not delete, it is just unlinked. + * A new API is provided to support non-production testing infrastructures, + allowing a client to cull all orphaned UIDs via ``DELETE`` on + ``<api>/reserved/uids/orphans``. Note that *no guarantees* of API + stability will ever be made for resources under ``reserved``. + (LP: #1420083) 3.0 beta 5 -- "Carve Away The Stone" @@ -122,6 +132,7 @@ REST section names via ``/3.0/system/configuration`` which returns a dictionary containing the ``http_etag`` and the section names as a sorted list under the ``sections`` key. The system configuration resource is read-only. + * Member resource JSON now include the ``member_id`` as a separate key. 3.0 beta 4 -- "Time and Motion" diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index ae4fe43a5..3ca4460b4 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -188,14 +188,13 @@ The first existing file found wins. * ``/etc/mailman.cfg`` * ``argv[0]/../../etc/mailman.cfg`` -Run the ``bin/mailman info`` command to see which configuration file Mailman -will use, and where it will put its database file. The first time you run -this, Mailman will also create any necessary run-time directories and log -files. +Run the ``mailman info`` command to see which configuration file Mailman will +use, and where it will put its database file. The first time you run this, +Mailman will also create any necessary run-time directories and log files. -Try ``bin/mailman --help`` for more details. You can use the commands -``bin/mailman start`` to start the runner subprocess daemons, and of course -``bin/mailman stop`` to stop them. +Try ``mailman --help`` for more details. You can use the commands +``mailman start`` to start the runner subprocess daemons, and of course +``mailman stop`` to stop them. Postorius, a web UI for administration and subscriber settings, is being developed as a separate, Django-based project. For now, the most flexible diff --git a/src/mailman/docs/WebUIin5.rst b/src/mailman/docs/WebUIin5.rst index 135f50484..bbcd7f194 100644 --- a/src/mailman/docs/WebUIin5.rst +++ b/src/mailman/docs/WebUIin5.rst @@ -56,7 +56,7 @@ directly on the PYTHONPATH. :: $(py2) cd mailman.client - $(py2) sudo python setup.py develop + $(py2) python setup.py develop $(py2) cd .. @@ -67,7 +67,7 @@ Postorius $(py2) bzr branch lp:postorius $(py2) cd postorius - $(py2) sudo python setup.py develop + $(py2) python setup.py develop Start the development server diff --git a/src/mailman/model/tests/test_uid.py b/src/mailman/model/tests/test_uid.py index d36fa4c3b..8f3b4af70 100644 --- a/src/mailman/model/tests/test_uid.py +++ b/src/mailman/model/tests/test_uid.py @@ -25,8 +25,11 @@ __all__ = [ import uuid import unittest +from mailman.config import config +from mailman.interfaces.usermanager import IUserManager from mailman.model.uid import UID from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -44,3 +47,41 @@ class TestUID(unittest.TestCase): my_uuid = uuid.uuid4() UID.record(my_uuid) self.assertRaises(ValueError, UID.record, my_uuid) + + def test_get_total_uid_count(self): + # The reserved REST API needs this. + for i in range(10): + UID.record(uuid.uuid4()) + self.assertEqual(UID.get_total_uid_count(), 10) + + def test_cull_orphan_uids(self): + # The reserved REST API needs to cull entries from the uid table that + # are not associated with actual entries in the user table. + manager = getUtility(IUserManager) + uids = set() + for i in range(10): + user = manager.create_user() + uids.add(user.user_id) + # The testing infrastructure does not record the UIDs for new user + # objects, so do that now to mimic the real system. + UID.record(user.user_id) + self.assertEqual(len(uids), 10) + # Now add some orphan uids. + orphans = set() + for i in range(100, 113): + uid = UID.record(uuid.UUID(int=i)) + orphans.add(uid.uid) + self.assertEqual(len(orphans), 13) + # Normally we wouldn't do a query in a test, since we'd want the model + # object to expose this, but we actually don't support exposing all + # the UIDs to the rest of Mailman. + all_uids = set(row[0] for row in config.db.store.query(UID.uid)) + self.assertEqual(all_uids, uids | orphans) + # Now, cull all the UIDs that aren't associated with users. Do use + # the model API for this. + UID.cull_orphans() + non_orphans = set(row[0] for row in config.db.store.query(UID.uid)) + self.assertEqual(uids, non_orphans) + # And all the users still exist. + non_orphans = set(user.user_id for user in manager.users) + self.assertEqual(uids, non_orphans) diff --git a/src/mailman/model/uid.py b/src/mailman/model/uid.py index c0d3e4d4d..0ff22438c 100644 --- a/src/mailman/model/uid.py +++ b/src/mailman/model/uid.py @@ -74,3 +74,20 @@ class UID(Model): if existing.count() != 0: raise ValueError(uid) return UID(uid) + + @staticmethod + @dbconnection + def get_total_uid_count(store): + return store.query(UID).count() + + @staticmethod + @dbconnection + def cull_orphans(store): + # Avoid circular imports. + from mailman.model.user import User + # Delete all uids in this table that are not associated with user + # rows. + results = store.query(UID).filter( + ~UID.uid.in_(store.query(User._user_id))) + for uid in results.all(): + store.delete(uid) diff --git a/src/mailman/rest/docs/addresses.rst b/src/mailman/rest/docs/addresses.rst index 8629bb8ae..fd3520be9 100644 --- a/src/mailman/rest/docs/addresses.rst +++ b/src/mailman/rest/docs/addresses.rst @@ -377,6 +377,7 @@ Elle can get her memberships for each of her email addresses. email: elle@example.com http_etag: "..." list_id: ant.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/4 @@ -386,6 +387,7 @@ Elle can get her memberships for each of her email addresses. email: elle@example.com http_etag: "..." list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/4 @@ -416,6 +418,7 @@ does not show up in the list of memberships for his other address. email: elle@example.com http_etag: "..." list_id: ant.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/4 @@ -425,6 +428,7 @@ does not show up in the list of memberships for his other address. email: elle@example.com http_etag: "..." list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/4 @@ -440,6 +444,7 @@ does not show up in the list of memberships for his other address. email: eperson@example.com http_etag: "..." list_id: bee.example.com + member_id: 3 role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/4 diff --git a/src/mailman/rest/docs/membership.rst b/src/mailman/rest/docs/membership.rst index b0b884d51..0343f40a1 100644 --- a/src/mailman/rest/docs/membership.rst +++ b/src/mailman/rest/docs/membership.rst @@ -46,6 +46,7 @@ the REST interface. email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -61,6 +62,7 @@ Bart's specific membership can be accessed directly: email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -76,6 +78,7 @@ the REST interface. email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -85,6 +88,7 @@ the REST interface. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -105,6 +109,7 @@ subscribes, she is returned first. email: aperson@example.com http_etag: ... list_id: bee.example.com + member_id: 3 role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 @@ -114,6 +119,7 @@ subscribes, she is returned first. email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -123,6 +129,7 @@ subscribes, she is returned first. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -147,6 +154,7 @@ User ids are different than member ids. email: aperson@example.com http_etag: ... list_id: ant.example.com + member_id: 4 role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 @@ -156,6 +164,7 @@ User ids are different than member ids. email: cperson@example.com http_etag: ... list_id: ant.example.com + member_id: 5 role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 @@ -165,6 +174,7 @@ User ids are different than member ids. email: aperson@example.com http_etag: ... list_id: bee.example.com + member_id: 3 role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 @@ -174,6 +184,7 @@ User ids are different than member ids. email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -183,6 +194,7 @@ User ids are different than member ids. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -200,6 +212,7 @@ We can also get just the members of a single mailing list. email: aperson@example.com http_etag: ... list_id: ant.example.com + member_id: 4 role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 @@ -209,6 +222,7 @@ We can also get just the members of a single mailing list. email: cperson@example.com http_etag: ... list_id: ant.example.com + member_id: 5 role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 @@ -234,6 +248,7 @@ page. email: aperson@example.com http_etag: ... list_id: ant.example.com + member_id: 4 role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 @@ -251,6 +266,7 @@ This works with members of a single list as well as with all members. email: aperson@example.com http_etag: ... list_id: ant.example.com + member_id: 4 role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 @@ -296,6 +312,7 @@ mailing list. email: dperson@example.com http_etag: ... list_id: ant.example.com + member_id: 6 role: moderator self_link: http://localhost:9001/3.0/members/6 user: http://localhost:9001/3.0/users/4 @@ -305,6 +322,7 @@ mailing list. email: aperson@example.com http_etag: ... list_id: ant.example.com + member_id: 4 role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 @@ -314,6 +332,7 @@ mailing list. email: cperson@example.com http_etag: ... list_id: ant.example.com + member_id: 5 role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 @@ -323,6 +342,7 @@ mailing list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 7 role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -332,6 +352,7 @@ mailing list. email: aperson@example.com http_etag: ... list_id: bee.example.com + member_id: 3 role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 @@ -341,6 +362,7 @@ mailing list. email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -350,6 +372,7 @@ mailing list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -367,6 +390,7 @@ We can access all the owners of a list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 7 role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -387,6 +411,7 @@ A specific member can always be referenced by their role and address. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 7 role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -403,6 +428,7 @@ example, we can search for all the memberships of a particular address. email: aperson@example.com http_etag: ... list_id: ant.example.com + member_id: 4 role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 @@ -412,6 +438,7 @@ example, we can search for all the memberships of a particular address. email: aperson@example.com http_etag: ... list_id: bee.example.com + member_id: 3 role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 @@ -430,6 +457,7 @@ Or, we can find all the memberships for a particular mailing list. email: aperson@example.com http_etag: ... list_id: bee.example.com + member_id: 3 role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 @@ -439,6 +467,7 @@ Or, we can find all the memberships for a particular mailing list. email: bperson@example.com http_etag: ... list_id: bee.example.com + member_id: 1 role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -448,6 +477,7 @@ Or, we can find all the memberships for a particular mailing list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -457,6 +487,7 @@ Or, we can find all the memberships for a particular mailing list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 7 role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -477,6 +508,7 @@ list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -486,6 +518,7 @@ list. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 7 role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -505,6 +538,7 @@ Or, we can find all the memberships for an address with a specific role. email: cperson@example.com http_etag: ... list_id: ant.example.com + member_id: 5 role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 @@ -514,6 +548,7 @@ Or, we can find all the memberships for an address with a specific role. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -534,6 +569,7 @@ Finally, we can search for a specific member given all three criteria. email: cperson@example.com http_etag: ... list_id: bee.example.com + member_id: 2 role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -583,6 +619,7 @@ Elly is now a known user, and a member of the mailing list. email: eperson@example.com http_etag: ... list_id: ant.example.com + member_id: 8 role: member self_link: http://localhost:9001/3.0/members/8 user: http://localhost:9001/3.0/users/5 @@ -625,6 +662,7 @@ list with her preferred address. email: gwen@example.com http_etag: "..." list_id: ant.example.com + member_id: 9 role: member self_link: http://localhost:9001/3.0/members/9 user: http://localhost:9001/3.0/users/6 @@ -649,6 +687,7 @@ the new address. email: gwen.person@example.com http_etag: "..." list_id: ant.example.com + member_id: 9 role: member self_link: http://localhost:9001/3.0/members/9 user: http://localhost:9001/3.0/users/6 @@ -715,6 +754,7 @@ Fred is getting MIME deliveries. email: fperson@example.com http_etag: "..." list_id: ant.example.com + member_id: 10 role: member self_link: http://localhost:9001/3.0/members/10 user: http://localhost:9001/3.0/users/7 @@ -738,6 +778,7 @@ This can be done by PATCH'ing his member with the `delivery_mode` parameter. email: fperson@example.com http_etag: "..." list_id: ant.example.com + member_id: 10 role: member self_link: http://localhost:9001/3.0/members/10 user: http://localhost:9001/3.0/users/7 @@ -757,6 +798,7 @@ If a PATCH request changes no attributes, nothing happens. email: fperson@example.com http_etag: "..." list_id: ant.example.com + member_id: 10 role: member self_link: http://localhost:9001/3.0/members/10 user: http://localhost:9001/3.0/users/7 @@ -800,6 +842,7 @@ addresses. email: herb@example.com http_etag: "..." list_id: ant.example.com + member_id: 11 role: member self_link: http://localhost:9001/3.0/members/11 user: http://localhost:9001/3.0/users/8 @@ -810,6 +853,7 @@ addresses. email: herb@example.com http_etag: "..." list_id: bee.example.com + member_id: 12 role: member self_link: http://localhost:9001/3.0/members/12 user: http://localhost:9001/3.0/users/8 @@ -867,6 +911,7 @@ his membership ids have not changed. email: hperson@example.com http_etag: "..." list_id: ant.example.com + member_id: 11 role: member self_link: http://localhost:9001/3.0/members/11 user: http://localhost:9001/3.0/users/8 @@ -876,6 +921,7 @@ his membership ids have not changed. email: hperson@example.com http_etag: "..." list_id: bee.example.com + member_id: 12 role: member self_link: http://localhost:9001/3.0/members/12 user: http://localhost:9001/3.0/users/8 diff --git a/src/mailman/rest/docs/queues.rst b/src/mailman/rest/docs/queues.rst index 861b6806f..df2753dc4 100644 --- a/src/mailman/rest/docs/queues.rst +++ b/src/mailman/rest/docs/queues.rst @@ -120,7 +120,7 @@ existing mailing list. content-length: 0 date: ... location: http://localhost:9001/3.0/lists/ant.example.com - server: WSGIServer/0.2 CPython/3.4.2 + server: WSGIServer/0.2 CPython/... status: 201 While list creation takes an FQDN list name, injecting a message to the queue diff --git a/src/mailman/rest/members.py b/src/mailman/rest/members.py index 925cd6718..a0b5d4f4e 100644 --- a/src/mailman/rest/members.py +++ b/src/mailman/rest/members.py @@ -58,13 +58,15 @@ class _MemberBase(CollectionMixin): # subscribed to will not have a user id. The user_id and the # member_id are UUIDs. We need to use the integer equivalent in the # URL. + member_id = member.member_id.int response = dict( list_id=member.list_id, email=member.address.email, role=role, address=path_to('addresses/{}'.format(member.address.email)), - self_link=path_to('members/{}'.format(member.member_id.int)), + self_link=path_to('members/{}'.format(member_id)), delivery_mode=member.delivery_mode, + member_id=member_id, ) # Add the user link if there is one. user = member.user diff --git a/src/mailman/rest/root.py b/src/mailman/rest/root.py index d4dca146e..0861a9a5b 100644 --- a/src/mailman/rest/root.py +++ b/src/mailman/rest/root.py @@ -29,10 +29,11 @@ from mailman.config import config from mailman.core.constants import system_preferences from mailman.core.system import system from mailman.interfaces.listmanager import IListManager +from mailman.model.uid import UID from mailman.rest.addresses import AllAddresses, AnAddress from mailman.rest.domains import ADomain, AllDomains from mailman.rest.helpers import ( - BadRequest, NotFound, child, etag, not_found, okay, path_to) + BadRequest, NotFound, child, etag, no_content, not_found, okay, path_to) from mailman.rest.lists import AList, AllLists, Styles from mailman.rest.members import AMember, AllMembers, FindMembers from mailman.rest.preferences import ReadOnlyPreferences @@ -42,6 +43,9 @@ from mailman.rest.users import AUser, AllUsers from zope.component import getUtility +SLASH = '/' + + class Root: """The RESTful root resource. @@ -110,6 +114,25 @@ class SystemConfiguration: okay(response, etag(resource)) +class Reserved: + """Top level API for reserved operations. + + Nothing under this resource should be considered part of the stable API. + The resources that appear here are purely for the support of external + non-production systems, such as testing infrastructures for cooperating + components. Use at your own risk. + """ + def __init__(self, segments): + self._resource_path = SLASH.join(segments) + + def on_delete(self, request, response): + if self._resource_path != 'uids/orphans': + not_found(response) + return + UID.cull_orphans() + no_content(response) + + class TopLevel: """Top level collections and entries.""" @@ -226,3 +249,8 @@ class TopLevel: return AQueueFile(segments[0], segments[1]), [] else: return BadRequest(), [] + + @child() + def reserved(self, request, segments): + """/<api>/reserved/[...]""" + return Reserved(segments), [] diff --git a/src/mailman/rest/tests/test_membership.py b/src/mailman/rest/tests/test_membership.py index e1bff833b..a77dea3b5 100644 --- a/src/mailman/rest/tests/test_membership.py +++ b/src/mailman/rest/tests/test_membership.py @@ -39,6 +39,12 @@ from urllib.error import HTTPError from zope.component import getUtility +def _set_preferred(user): + preferred = list(user.addresses)[0] + preferred.verified_on = now() + user.preferred_address = preferred + + class TestMembership(unittest.TestCase): layer = RESTLayer @@ -98,6 +104,36 @@ class TestMembership(unittest.TestCase): self.assertEqual(cm.exception.code, 409) self.assertEqual(cm.exception.reason, b'Member already subscribed') + def test_add_member_with_mixed_case_email(self): + # LP: #1425359 - Mailman is case-perserving, case-insensitive. This + # test subscribes the lower case address and ensures the original mixed + # case address can't be subscribed. + with transaction(): + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/members', { + 'list_id': 'test.example.com', + 'subscriber': 'ANNE@example.com', + }) + self.assertEqual(cm.exception.code, 409) + self.assertEqual(cm.exception.reason, b'Member already subscribed') + + def test_add_member_with_lower_case_email(self): + # LP: #1425359 - Mailman is case-perserving, case-insensitive. This + # test subscribes the mixed case address and ensures the lower cased + # address can't be added. + with transaction(): + anne = self._usermanager.create_address('ANNE@example.com') + self._mlist.subscribe(anne) + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/members', { + 'list_id': 'test.example.com', + 'subscriber': 'anne@example.com', + }) + self.assertEqual(cm.exception.code, 409) + self.assertEqual(cm.exception.reason, b'Member already subscribed') + def test_join_with_invalid_delivery_mode(self): with self.assertRaises(HTTPError) as cm: call_api('http://localhost:9001/3.0/members', { @@ -129,9 +165,7 @@ class TestMembership(unittest.TestCase): def test_join_as_user_with_preferred_address(self): with transaction(): anne = self._usermanager.create_user('anne@example.com') - preferred = list(anne.addresses)[0] - preferred.verified_on = now() - anne.preferred_address = preferred + _set_preferred(anne) self._mlist.subscribe(anne) content, response = call_api('http://localhost:9001/3.0/members') self.assertEqual(response.status, 200) @@ -150,9 +184,7 @@ class TestMembership(unittest.TestCase): def test_member_changes_preferred_address(self): with transaction(): anne = self._usermanager.create_user('anne@example.com') - preferred = list(anne.addresses)[0] - preferred.verified_on = now() - anne.preferred_address = preferred + _set_preferred(anne) self._mlist.subscribe(anne) # Take a look at Anne's current membership. content, response = call_api('http://localhost:9001/3.0/members') diff --git a/src/mailman/rest/tests/test_root.py b/src/mailman/rest/tests/test_root.py index 6d10fc635..905461e46 100644 --- a/src/mailman/rest/tests/test_root.py +++ b/src/mailman/rest/tests/test_root.py @@ -123,3 +123,11 @@ class TestRoot(unittest.TestCase): self.assertEqual(content['title'], '401 Unauthorized') self.assertEqual(content['description'], 'User is not authorized for the REST API') + + def test_reserved_bad_subpath(self): + # Only <api>/reserved/uids/orphans is a defined resource. DELETEing + # anything else gives a 404. + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/reserved/uids/assigned', + method='DELETE') + self.assertEqual(cm.exception.code, 404) diff --git a/src/mailman/rest/tests/test_uids.py b/src/mailman/rest/tests/test_uids.py new file mode 100644 index 000000000..6c31a8aa4 --- /dev/null +++ b/src/mailman/rest/tests/test_uids.py @@ -0,0 +1,76 @@ +# Copyright (C) 2015 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test deletion of orphaned UIDs. + +There is no doctest for this functionality, since it's only useful for testing +of external clients of the REST API. +""" + +__all__ = [ + 'TestUIDs', + ] + + +import unittest + +from mailman.config import config +from mailman.database.transaction import transaction +from mailman.interfaces.usermanager import IUserManager +from mailman.model.uid import UID +from mailman.testing.helpers import call_api +from mailman.testing.layers import RESTLayer +from zope.component import getUtility + + + +class TestUIDs(unittest.TestCase): + layer = RESTLayer + + def test_delete_orphans(self): + # When users are deleted, their UIDs are generally not deleted. We + # never delete rows from that table in order to guarantee no + # duplicates. However, some external testing frameworks want to be + # able to reset the UID table, so they can use this interface to do + # so. See LP: #1420083. + # + # Create some users. + manager = getUtility(IUserManager) + users_by_uid = {} + with transaction(): + for i in range(10): + user = manager.create_user() + users_by_uid[user.user_id] = user + # The testing infrastructure does not record the UIDs for new + # user options, so do that now to mimic the real system. + UID.record(user.user_id) + # We now have 10 unique uids. + self.assertEqual(len(users_by_uid), 10) + # Now delete all the users. + with transaction(): + for user in list(users_by_uid.values()): + manager.delete_user(user) + # There are still 10 unique uids in the database. + self.assertEqual(UID.get_total_uid_count(), 10) + # Cull the orphan UIDs. + content, response = call_api( + 'http://localhost:9001/3.0/reserved/uids/orphans', + method='DELETE') + self.assertEqual(response.status, 204) + # Now there are no uids in the table. + config.db.abort() + self.assertEqual(UID.get_total_uid_count(), 0) |