From dbde6231ec897379ed38ed4cd015b8ab20ed5fa1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 16 Jul 2016 15:44:07 -0400 Subject: New template system. Closes #249 The new template system is introduced for API 3.1. See ``src/mailman/rest/docs/templates.rst`` for details. --- setup.py | 1 + src/mailman/app/bounces.py | 33 +- src/mailman/app/docs/moderator.rst | 6 +- src/mailman/app/docs/pipelines.rst | 12 +- src/mailman/app/membership.py | 13 +- src/mailman/app/moderator.py | 36 +- src/mailman/app/notifications.py | 62 +-- src/mailman/app/registrar.py | 26 +- src/mailman/app/subscriptions.py | 13 +- src/mailman/app/templates.py | 104 ---- src/mailman/app/tests/test_bounces.py | 53 +- src/mailman/app/tests/test_notifications.py | 32 +- src/mailman/app/tests/test_registrar.py | 3 +- src/mailman/app/tests/test_subscriptions.py | 3 +- src/mailman/app/tests/test_templates.py | 125 ----- src/mailman/archiving/docs/common.rst | 4 +- src/mailman/archiving/mhonarc.py | 12 +- src/mailman/chains/hold.py | 48 +- src/mailman/commands/cli_lists.py | 18 +- src/mailman/commands/docs/create.rst | 16 +- src/mailman/commands/docs/info.rst | 1 + src/mailman/commands/tests/test_lists.py | 3 +- src/mailman/config/config.py | 4 +- src/mailman/config/configure.zcml | 14 +- src/mailman/config/schema.cfg | 5 + src/mailman/core/runner.py | 2 +- src/mailman/core/switchboard.py | 2 +- src/mailman/database/alembic/env.py | 4 +- .../versions/fa0d96e28631_template_manager.py | 178 +++++++ src/mailman/database/base.py | 2 +- src/mailman/database/tests/test_migrations.py | 156 ++++++ src/mailman/docs/NEWS.rst | 2 + src/mailman/handlers/acknowledge.py | 24 +- src/mailman/handlers/decorate.py | 63 +-- src/mailman/handlers/docs/acknowledge.rst | 8 - src/mailman/handlers/docs/decorate.rst | 26 +- src/mailman/handlers/docs/rfc-2369.rst | 24 +- src/mailman/handlers/replybot.py | 3 +- src/mailman/handlers/rfc_2369.py | 7 +- src/mailman/handlers/tests/test_decorate.py | 16 +- src/mailman/handlers/tests/test_rfc_2369.py | 7 +- src/mailman/interfaces/cache.py | 66 +++ src/mailman/interfaces/domain.py | 25 +- src/mailman/interfaces/mailinglist.py | 210 +------- src/mailman/interfaces/member.py | 18 +- src/mailman/interfaces/template.py | 199 ++++++++ src/mailman/interfaces/templates.py | 40 -- src/mailman/model/cache.py | 160 ++++++ src/mailman/model/docs/domains.rst | 42 +- src/mailman/model/domain.py | 33 +- src/mailman/model/mailinglist.py | 25 +- src/mailman/model/member.py | 21 +- src/mailman/model/template.py | 202 ++++++++ src/mailman/model/tests/test_cache.py | 110 +++++ src/mailman/model/tests/test_template.py | 286 +++++++++++ src/mailman/mta/deliver.py | 8 +- src/mailman/mta/docs/decorating.rst | 21 +- src/mailman/mta/tests/test_delivery.py | 15 +- src/mailman/mta/verp.py | 2 +- src/mailman/rest/docs/__init__.py | 97 +++- src/mailman/rest/docs/domains.rst | 38 +- src/mailman/rest/docs/listconf.rst | 13 +- src/mailman/rest/docs/systemconf.rst | 1 + src/mailman/rest/docs/templates.rst | 539 +++++++++++++++++++++ src/mailman/rest/domains.py | 25 +- src/mailman/rest/header_matches.py | 6 +- src/mailman/rest/listconf.py | 91 +++- src/mailman/rest/lists.py | 22 +- src/mailman/rest/root.py | 17 + src/mailman/rest/sub_moderation.py | 2 +- src/mailman/rest/tests/test_bans.py | 5 + src/mailman/rest/tests/test_domains.py | 222 ++++++++- src/mailman/rest/tests/test_listconf.py | 23 +- src/mailman/rest/tests/test_lists.py | 232 +++++++++ src/mailman/rest/tests/test_root.py | 211 ++++++++ src/mailman/rest/tests/test_systemconf.py | 1 + src/mailman/rest/uris.py | 203 ++++++++ src/mailman/rest/validator.py | 3 +- src/mailman/rest/wsgiapp.py | 3 +- src/mailman/runners/digest.py | 59 +-- src/mailman/runners/docs/digester.rst | 12 +- src/mailman/runners/tests/test_digest.py | 27 ++ src/mailman/styles/base.py | 10 - src/mailman/templates/en/adminaddrchgack.txt | 4 - src/mailman/templates/en/admindbdetails.html | 65 --- src/mailman/templates/en/admindbpreamble.html | 10 - src/mailman/templates/en/admindbsummary.html | 14 - src/mailman/templates/en/adminsubscribeack.txt | 1 - src/mailman/templates/en/adminunsubscribeack.txt | 1 - src/mailman/templates/en/admlogin.html | 39 -- src/mailman/templates/en/approve.txt | 15 - src/mailman/templates/en/article.html | 50 -- src/mailman/templates/en/bounce.txt | 13 - src/mailman/templates/en/checkdbs.txt | 7 - src/mailman/templates/en/confirm.txt | 17 - src/mailman/templates/en/convert.txt | 34 -- src/mailman/templates/en/cronpass.txt | 19 - src/mailman/templates/en/disabled.txt | 25 - .../templates/en/domain:admin:notice:new-list.txt | 10 + src/mailman/templates/en/emptyarchive.html | 15 - src/mailman/templates/en/footer-generic.txt | 4 - src/mailman/templates/en/headfoot.html | 28 -- src/mailman/templates/en/help.txt | 31 +- src/mailman/templates/en/invite.txt | 4 - .../templates/en/list:admin:action:post.txt | 13 + .../templates/en/list:admin:action:subscribe.txt | 5 + .../templates/en/list:admin:action:unsubscribe.txt | 5 + .../templates/en/list:admin:notice:subscribe.txt | 1 + .../en/list:admin:notice:unrecognized.txt | 4 + .../templates/en/list:admin:notice:unsubscribe.txt | 1 + .../templates/en/list:member:digest:masthead.txt | 12 + .../templates/en/list:member:generic:footer.txt | 3 + .../templates/en/list:user:action:confirm.txt | 17 + .../templates/en/list:user:action:unsubscribe.txt | 19 + src/mailman/templates/en/list:user:notice:hold.txt | 12 + .../en/list:user:notice:no-more-today.txt | 9 + src/mailman/templates/en/list:user:notice:post.txt | 5 + .../templates/en/list:user:notice:probe.txt | 14 + .../templates/en/list:user:notice:refuse.txt | 13 + .../templates/en/list:user:notice:welcome.txt | 15 + src/mailman/templates/en/listinfo.html | 143 ------ src/mailman/templates/en/masthead.txt | 13 - src/mailman/templates/en/newlist.txt | 18 - src/mailman/templates/en/nomoretoday.txt | 9 - src/mailman/templates/en/options.html | 316 ------------ src/mailman/templates/en/postack.txt | 8 - src/mailman/templates/en/postauth.txt | 13 - src/mailman/templates/en/postheld.txt | 12 - src/mailman/templates/en/private.html | 43 -- src/mailman/templates/en/probe.txt | 20 - src/mailman/templates/en/refuse.txt | 13 - src/mailman/templates/en/roster.html | 52 -- src/mailman/templates/en/subauth.txt | 5 - src/mailman/templates/en/subscribe.html | 8 - src/mailman/templates/en/unrecognized.txt | 7 - src/mailman/templates/en/unsub.txt | 23 - src/mailman/templates/en/unsubauth.txt | 11 - src/mailman/templates/en/userpass.txt | 24 - src/mailman/templates/en/welcome.txt | 25 - src/mailman/testing/documentation.py | 7 +- src/mailman/testing/helpers.py | 25 +- src/mailman/testing/layers.py | 6 +- src/mailman/testing/mailman-fr.mo | Bin 1992 -> 1893 bytes src/mailman/testing/mailman-fr.po | 12 +- src/mailman/utilities/i18n.py | 45 -- src/mailman/utilities/importer.py | 52 +- src/mailman/utilities/protocols.py | 101 ++++ src/mailman/utilities/string.py | 31 +- src/mailman/utilities/tests/test_import.py | 78 +-- src/mailman/utilities/tests/test_protocols.py | 174 +++++++ src/mailman/utilities/tests/test_templates.py | 64 +-- 151 files changed, 4022 insertions(+), 2406 deletions(-) delete mode 100644 src/mailman/app/templates.py delete mode 100644 src/mailman/app/tests/test_templates.py create mode 100644 src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py create mode 100644 src/mailman/interfaces/cache.py create mode 100644 src/mailman/interfaces/template.py delete mode 100644 src/mailman/interfaces/templates.py create mode 100644 src/mailman/model/cache.py create mode 100644 src/mailman/model/template.py create mode 100644 src/mailman/model/tests/test_cache.py create mode 100644 src/mailman/model/tests/test_template.py create mode 100644 src/mailman/rest/docs/templates.rst create mode 100644 src/mailman/rest/uris.py delete mode 100644 src/mailman/templates/en/adminaddrchgack.txt delete mode 100644 src/mailman/templates/en/admindbdetails.html delete mode 100644 src/mailman/templates/en/admindbpreamble.html delete mode 100644 src/mailman/templates/en/admindbsummary.html delete mode 100644 src/mailman/templates/en/adminsubscribeack.txt delete mode 100644 src/mailman/templates/en/adminunsubscribeack.txt delete mode 100644 src/mailman/templates/en/admlogin.html delete mode 100644 src/mailman/templates/en/approve.txt delete mode 100644 src/mailman/templates/en/article.html delete mode 100644 src/mailman/templates/en/bounce.txt delete mode 100644 src/mailman/templates/en/checkdbs.txt delete mode 100644 src/mailman/templates/en/confirm.txt delete mode 100644 src/mailman/templates/en/convert.txt delete mode 100644 src/mailman/templates/en/cronpass.txt delete mode 100644 src/mailman/templates/en/disabled.txt create mode 100644 src/mailman/templates/en/domain:admin:notice:new-list.txt delete mode 100644 src/mailman/templates/en/emptyarchive.html delete mode 100644 src/mailman/templates/en/footer-generic.txt delete mode 100644 src/mailman/templates/en/headfoot.html create mode 100644 src/mailman/templates/en/list:admin:action:post.txt create mode 100644 src/mailman/templates/en/list:admin:action:subscribe.txt create mode 100644 src/mailman/templates/en/list:admin:action:unsubscribe.txt create mode 100644 src/mailman/templates/en/list:admin:notice:subscribe.txt create mode 100644 src/mailman/templates/en/list:admin:notice:unrecognized.txt create mode 100644 src/mailman/templates/en/list:admin:notice:unsubscribe.txt create mode 100644 src/mailman/templates/en/list:member:digest:masthead.txt create mode 100644 src/mailman/templates/en/list:member:generic:footer.txt create mode 100644 src/mailman/templates/en/list:user:action:confirm.txt create mode 100644 src/mailman/templates/en/list:user:action:unsubscribe.txt create mode 100644 src/mailman/templates/en/list:user:notice:hold.txt create mode 100644 src/mailman/templates/en/list:user:notice:no-more-today.txt create mode 100644 src/mailman/templates/en/list:user:notice:post.txt create mode 100644 src/mailman/templates/en/list:user:notice:probe.txt create mode 100644 src/mailman/templates/en/list:user:notice:refuse.txt create mode 100644 src/mailman/templates/en/list:user:notice:welcome.txt delete mode 100644 src/mailman/templates/en/listinfo.html delete mode 100644 src/mailman/templates/en/masthead.txt delete mode 100644 src/mailman/templates/en/newlist.txt delete mode 100644 src/mailman/templates/en/nomoretoday.txt delete mode 100644 src/mailman/templates/en/options.html delete mode 100644 src/mailman/templates/en/postack.txt delete mode 100644 src/mailman/templates/en/postauth.txt delete mode 100644 src/mailman/templates/en/postheld.txt delete mode 100644 src/mailman/templates/en/private.html delete mode 100644 src/mailman/templates/en/probe.txt delete mode 100644 src/mailman/templates/en/refuse.txt delete mode 100644 src/mailman/templates/en/roster.html delete mode 100644 src/mailman/templates/en/subauth.txt delete mode 100644 src/mailman/templates/en/subscribe.html delete mode 100644 src/mailman/templates/en/unrecognized.txt delete mode 100644 src/mailman/templates/en/unsub.txt delete mode 100644 src/mailman/templates/en/unsubauth.txt delete mode 100644 src/mailman/templates/en/userpass.txt delete mode 100644 src/mailman/templates/en/welcome.txt create mode 100644 src/mailman/utilities/protocols.py create mode 100644 src/mailman/utilities/tests/test_protocols.py diff --git a/setup.py b/setup.py index a585e2f9e..3fc6cbe98 100644 --- a/setup.py +++ b/setup.py @@ -114,6 +114,7 @@ case second `m'. Any other spelling is incorrect.""", 'lazr.smtptest', 'nose2', 'passlib', + 'requests', 'sqlalchemy', 'zope.component', 'zope.configuration', diff --git a/src/mailman/app/bounces.py b/src/mailman/app/bounces.py index 977b21038..98076dc9b 100644 --- a/src/mailman/app/bounces.py +++ b/src/mailman/app/bounces.py @@ -32,9 +32,9 @@ from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.listmanager import IListManager from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.subscriptions import ISubscriptionService +from mailman.interfaces.template import ITemplateLoader from mailman.utilities.email import split_email -from mailman.utilities.i18n import make -from mailman.utilities.string import oneline +from mailman.utilities.string import expand, oneline, wrap from string import Template from zope.component import getUtility from zope.interface import implementer @@ -184,12 +184,19 @@ def send_probe(member, msg): """ mlist = getUtility(IListManager).get_by_list_id( member.mailing_list.list_id) - text = make('probe.txt', mlist, member.preferred_language.code, - listname=mlist.fqdn_listname, - address=member.address.email, - optionsurl=member.options_url, - owneraddr=mlist.owner_address, - ) + template = getUtility(ITemplateLoader).get( + 'list:user:notice:probe', mlist, + language=member.preferred_language.code, + # For backward compatibility. + code=member.preferred_language.code, + ) + text = wrap(expand(template, mlist, dict( + sender_email=member.subscriber.email, + # For backward compatibility. + address=member.address.email, + email=member.address.email, + owneraddr=mlist.owner_address, + ))) message_id = msg['message-id'] if isinstance(message_id, bytes): message_id = message_id.decode('ascii') @@ -240,11 +247,11 @@ def maybe_forward(mlist, msg): # The notification is either going to go to the list's administrators # (owners and moderators), or to the site administrators. Most of the # notification is exactly the same in either case. - adminurl = mlist.script_url('admin') + '/bounce' subject = _('Uncaught bounce notification') - text = MIMEText( - make('unrecognized.txt', mlist, adminurl=adminurl), - _charset=mlist.preferred_language.charset) + template = getUtility(ITemplateLoader).get( + 'list:admin:notice:unrecognized', mlist) + text = expand(template, mlist) + text_part = MIMEText(text, _charset=mlist.preferred_language.charset) attachment = MIMEMessage(msg) if (mlist.forward_unrecognized_bounces_to is UnrecognizedBounceDisposition.administrators): @@ -258,6 +265,6 @@ def maybe_forward(mlist, msg): # Create the notification and send it. notice = OwnerNotification(mlist, subject, **keywords) notice.set_type('multipart/mixed') - notice.attach(text) + notice.attach(text_part) notice.attach(attachment) notice.send(mlist) diff --git a/src/mailman/app/docs/moderator.rst b/src/mailman/app/docs/moderator.rst index f4339667f..988e8bebc 100644 --- a/src/mailman/app/docs/moderator.rst +++ b/src/mailman/app/docs/moderator.rst @@ -355,9 +355,9 @@ Jeff is a member of the mailing list, and chooses to unsubscribe. Your authorization is required for a mailing list unsubscription request approval: - By: jeff@example.org - From: ant@example.com - ... + For: jeff@example.org + List: ant@example.com + Membership changes diff --git a/src/mailman/app/docs/pipelines.rst b/src/mailman/app/docs/pipelines.rst index 46880b225..4022251f7 100644 --- a/src/mailman/app/docs/pipelines.rst +++ b/src/mailman/app/docs/pipelines.rst @@ -49,14 +49,12 @@ etc. Precedence: list Subject: [Test] My first post List-Id: - Archived-At: - List-Archive: + Archived-At: + List-Archive: List-Help: List-Post: - List-Subscribe: , - - List-Unsubscribe: , - + List-Subscribe: + List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit @@ -65,7 +63,6 @@ etc. _______________________________________________ Test mailing list test@example.com - http://lists.example.com/listinfo/test@example.com The message metadata has information about recipients and other stuff. @@ -139,7 +136,6 @@ delivered to end recipients. _______________________________________________ Test mailing list test@example.com - http://lists.example.com/listinfo/test@example.com >>> dump_msgdata(messages[0].msgdata) _parsemsg : False diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index 8b4d7ff30..534eed15d 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -29,9 +29,10 @@ from mailman.interfaces.bans import IBanManager from mailman.interfaces.member import ( AlreadySubscribedError, MemberRole, MembershipIsBannedError, NotAMemberError, SubscriptionEvent) +from mailman.interfaces.template import ITemplateLoader from mailman.interfaces.user import IUser from mailman.interfaces.usermanager import IUserManager -from mailman.utilities.i18n import make +from mailman.utilities.string import expand from zope.component import getUtility @@ -132,11 +133,11 @@ def delete_member(mlist, email, admin_notif=None, userack=None): user = getUtility(IUserManager).get_user(email) display_name = user.display_name subject = _('$mlist.display_name unsubscription notification') - text = make('adminunsubscribeack.txt', - mailing_list=mlist, - listname=mlist.display_name, - member=formataddr((display_name, email)), - ) + text = expand(getUtility(ITemplateLoader).get( + 'list:admin:notice:unsubscribe', mlist), + mlist, dict( + member=formataddr((display_name, email)), + )) msg = OwnerNotification(mlist, subject, text, roster=mlist.administrators) msg.send(mlist) diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index 2f8c19cfc..8ce631941 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -31,8 +31,9 @@ from mailman.interfaces.listmanager import ListDeletingEvent from mailman.interfaces.member import NotAMemberError from mailman.interfaces.messages import IMessageStore from mailman.interfaces.requests import IListRequests, RequestType +from mailman.interfaces.template import ITemplateLoader from mailman.utilities.datetime import now -from mailman.utilities.i18n import make +from mailman.utilities.string import expand, wrap from zope.component import getUtility @@ -190,12 +191,14 @@ def hold_unsubscription(mlist, email): if mlist.admin_immed_notify: subject = _( 'New unsubscription request from $mlist.display_name by $email') - text = make('unsubauth.txt', - mailing_list=mlist, - email=email, - listname=mlist.fqdn_listname, - admindb_url=mlist.script_url('admindb'), - ) + template = getUtility(ITemplateLoader).get( + 'list:admin:action:unsubscribe', mlist) + text = wrap(expand(template, mlist, dict( + # For backward compatibility. + mailing_list=mlist, + member=email, + email=email, + ))) # This message should appear to come from the -owner so as # to avoid any useless bounce processing. msg = UserNotification( @@ -230,7 +233,7 @@ def handle_unsubscription(mlist, id, action, comment=None): pass slog.info('%s: deleted %s', mlist.fqdn_listname, email) else: - raise AssertionError('Unexpected action: {0}'.format(action)) + raise AssertionError('Unexpected action: {}'.format(action)) # Delete the request from the database. requestdb.delete_request(id) @@ -246,14 +249,15 @@ def send_rejection(mlist, request, recip, comment, origmsg=None, lang=None): lang = (mlist.preferred_language if member is None else member.preferred_language) - text = make('refuse.txt', - mailing_list=mlist, - language=lang.code, - listname=mlist.fqdn_listname, - request=request, - reason=comment, - adminaddr=mlist.owner_address, - ) + template = getUtility(ITemplateLoader).get( + 'list:user:notice:refuse', mlist) + text = wrap(expand(template, mlist, dict( + language=lang.code, + reason=comment, + # For backward compatibility. + request=request, + adminaddr=mlist.owner_address, + ))) with _.using(lang.code): # add in original message, but not wrap/filled if origmsg: diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py index 04a56b55d..11a21a379 100644 --- a/src/mailman/app/notifications.py +++ b/src/mailman/app/notifications.py @@ -26,33 +26,14 @@ from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import OwnerNotification, UserNotification from mailman.interfaces.member import DeliveryMode -from mailman.interfaces.templates import ITemplateLoader -from mailman.utilities.i18n import make +from mailman.interfaces.template import ITemplateLoader from mailman.utilities.string import expand, wrap -from urllib.error import URLError from zope.component import getUtility log = logging.getLogger('mailman.error') -def _get_message(uri_template, mlist, language): - if not uri_template: - return '' - try: - uri = expand(uri_template, dict( - listname=mlist.fqdn_listname, - language=language.code, - )) - message = getUtility(ITemplateLoader).get(uri) - except URLError: - log.exception('Message URI not found ({0}): {1}'.format( - mlist.fqdn_listname, uri_template)) - return '' - else: - return wrap(message) - - @public def send_welcome_message(mlist, member, language, text=''): """Send a welcome message to a subscriber. @@ -67,29 +48,18 @@ def send_welcome_message(mlist, member, language, text=''): :param language: The language of the response. :type language: ILanguage """ - welcome_message = _get_message(mlist.welcome_message_uri, mlist, language) - options_url = member.options_url - # Try to find a non-empty display name. We first look at the directly - # subscribed record, which will either be the address or the user. That's - # handled automatically by going through member.subscriber. If that - # doesn't give us something useful, try whatever user is linked to the - # subscriber. - if member.subscriber.display_name: - display_name = member.subscriber.display_name - # If an unlinked address is subscribed tehre will be no .user. - elif member.user is not None and member.user.display_name: - display_name = member.user.display_name - else: - display_name = '' + welcome_message = wrap(getUtility(ITemplateLoader).get( + 'list:user:notice:welcome', mlist, language=language.code)) + display_name = member.display_name # Get the text from the template. - text = expand(welcome_message, dict( + text = expand(welcome_message, mlist, dict( + user_name=display_name, + user_email=member.address.email, + # For backward compatibility. + user_address=member.address.email, fqdn_listname=mlist.fqdn_listname, list_name=mlist.display_name, - listinfo_uri=mlist.script_url('listinfo'), list_requests=mlist.request_address, - user_name=display_name, - user_address=member.address.email, - user_options_uri=options_url, )) digmode = ('' # noqa if member.delivery_mode is DeliveryMode.regular @@ -117,8 +87,8 @@ def send_goodbye_message(mlist, address, language): :param language: the language of the response :type language: string """ - goodbye_message = _get_message(mlist.goodbye_message_uri, - mlist, language) + goodbye_message = wrap(getUtility(ITemplateLoader).get( + 'list:user:notice:goodbye', mlist, language=language.code)) msg = UserNotification( address, mlist.bounces_address, _('You have been unsubscribed from the $mlist.display_name ' @@ -140,10 +110,10 @@ def send_admin_subscription_notice(mlist, address, display_name): """ with _.using(mlist.preferred_language.code): subject = _('$mlist.display_name subscription notification') - text = make('adminsubscribeack.txt', - mailing_list=mlist, - listname=mlist.display_name, - member=formataddr((display_name, address)), - ) + text = expand( + getUtility(ITemplateLoader).get('list:admin:notice:subscribe', mlist), + mlist, dict( + member=formataddr((display_name, address)), + )) msg = OwnerNotification(mlist, subject, text, roster=mlist.administrators) msg.send(mlist) diff --git a/src/mailman/app/registrar.py b/src/mailman/app/registrar.py index 541006f4b..7cefbd518 100644 --- a/src/mailman/app/registrar.py +++ b/src/mailman/app/registrar.py @@ -21,13 +21,13 @@ import logging from mailman import public from mailman.app.subscriptions import SubscriptionWorkflow -from mailman.core.i18n import _ from mailman.database.transaction import flush from mailman.email.message import UserNotification from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.registrar import ConfirmationNeededEvent, IRegistrar -from mailman.interfaces.templates import ITemplateLoader +from mailman.interfaces.template import ITemplateLoader from mailman.interfaces.workflow import IWorkflowStateManager +from mailman.utilities.string import expand from zope.component import getUtility from zope.interface import implementer @@ -84,18 +84,22 @@ def handle_ConfirmationNeededEvent(event): # encode the token, they can reply to the robot and keep the token in # the Subject header, or they can click on the URL in the body of the # message and confirm through the web. - subject = 'confirm ' + event.token + subject = 'confirm {}'.format(event.token) confirm_address = event.mlist.confirm_address(event.token) - # For i18n interpolation. - confirm_url = event.mlist.domain.confirm_url(event.token) # noqa email_address = event.email - domain_name = event.mlist.domain.mail_host # noqa - contact_address = event.mlist.owner_address # noqa # Send a verification email to the address. template = getUtility(ITemplateLoader).get( - 'mailman:///{}/{}/confirm.txt'.format( - event.mlist.fqdn_listname, - event.mlist.preferred_language.code)) - text = _(template) + 'list:user:action:confirm', event.mlist) + text = expand(template, event.mlist, dict( + token=event.token, + subject=subject, + confirm_email=confirm_address, + user_email=email_address, + # For backward compatibility. + confirm_address=confirm_address, + email_address=email_address, + domain_name=event.mlist.domain.mail_host, + contact_address=event.mlist.owner_address, + )) msg = UserNotification(email_address, confirm_address, subject, text) msg.send(event.mlist, add_precedence=False) diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index e360d9615..3042623d1 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -36,11 +36,12 @@ from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.registrar import ConfirmationNeededEvent from mailman.interfaces.subscriptions import ( ISubscriptionService, SubscriptionPendingError, TokenOwner) +from mailman.interfaces.template import ITemplateLoader from mailman.interfaces.user import IUser from mailman.interfaces.usermanager import IUserManager from mailman.interfaces.workflow import IWorkflowStateManager from mailman.utilities.datetime import now -from mailman.utilities.i18n import make +from mailman.utilities.string import expand, wrap from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -257,11 +258,11 @@ class SubscriptionWorkflow(Workflow): 'from $self.address.email') username = formataddr( (self.subscriber.display_name, self.address.email)) - text = make('subauth.txt', - mailing_list=self.mlist, - username=username, - listname=self.mlist.fqdn_listname, - ) + template = getUtility(ITemplateLoader).get( + 'list:admin:action:subscribe', self.mlist) + text = wrap(expand(template, self.mlist, dict( + member=username, + ))) # This message should appear to come from the -owner so as # to avoid any useless bounce processing. msg = UserNotification( diff --git a/src/mailman/app/templates.py b/src/mailman/app/templates.py deleted file mode 100644 index c01b9af47..000000000 --- a/src/mailman/app/templates.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (C) 2012-2016 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Template loader.""" - -from contextlib import closing -from mailman import public -from mailman.interfaces.languages import ILanguageManager -from mailman.interfaces.listmanager import IListManager -from mailman.interfaces.templates import ITemplateLoader -from mailman.utilities.i18n import TemplateNotFoundError, find -from urllib.error import URLError -from urllib.parse import urlparse -from urllib.request import BaseHandler, build_opener, install_opener, urlopen -from urllib.response import addinfourl -from zope.component import getUtility -from zope.interface import implementer - - -class MailmanHandler(BaseHandler): - # Handle internal mailman: URLs. - def mailman_open(self, req): - list_manager = getUtility(IListManager) - # Parse urls of the form: - # - # mailman:///// - # - # where only the template name is required. - mlist = code = template = None - # Parse the full requested URL and be sure it's something we handle. - original_url = req.get_full_url() - parsed = urlparse(original_url) - assert parsed.scheme == 'mailman' - # The path can contain one, two, or three components. Since no empty - # path components are legal, filter them out. - parts = [p for p in parsed.path.split('/') if p] - if len(parts) == 0: - raise URLError('No template specified') - elif len(parts) == 1: - template = parts[0] - elif len(parts) == 2: - part0, template = parts - # Is part0 a language code or a mailing list? This is rather - # tricky because if it's a mailing list, it could be a list-id and - # that will contain dots, as could the language code. - language = getUtility(ILanguageManager).get(part0) - if language is None: - # part0 must be a fqdn-listname or list-id. - mlist = (list_manager.get(part0) - if '@' in part0 else - list_manager.get_by_list_id(part0)) - if mlist is None: - raise URLError('Bad language or list name') - else: - code = language.code - elif len(parts) == 3: - part0, code, template = parts - # part0 could be an fqdn-listname or a list-id. - mlist = (getUtility(IListManager).get(part0) - if '@' in part0 else - getUtility(IListManager).get_by_list_id(part0)) - if mlist is None: - raise URLError('Missing list') - language = getUtility(ILanguageManager).get(code) - if language is None: - raise URLError('No such language') - code = language.code - else: - raise URLError('No such file') - # Find the template, mutating any missing template exception. - try: - path, fp = find(template, mlist, code) - except TemplateNotFoundError: - raise URLError('No such file') - return addinfourl(fp, {}, original_url) - - -@public -@implementer(ITemplateLoader) -class TemplateLoader: - """Loader of templates, with caching and support for mailman:// URIs.""" - - def __init__(self): - opener = build_opener(MailmanHandler()) - install_opener(opener) - - def get(self, uri): - """See `ITemplateLoader`.""" - with closing(urlopen(uri)) as fp: - return fp.read() diff --git a/src/mailman/app/tests/test_bounces.py b/src/mailman/app/tests/test_bounces.py index ffc1cf2b4..685f02907 100644 --- a/src/mailman/app/tests/test_bounces.py +++ b/src/mailman/app/tests/test_bounces.py @@ -177,6 +177,7 @@ class TestSendProbe(unittest.TestCase): """Test sending of the probe message.""" layer = ConfigLayer + maxDiff = None def setUp(self): self._mlist = create_list('test@example.com') @@ -240,13 +241,24 @@ Message-ID: self.assertEqual(notice.get_content_type(), 'text/plain') # The interesting bits are the parts that have been interpolated into # the message. For now the best we can do is know that the - # interpolation values appear in the message. When Python 2.7 is our - # minimum requirement, we can use assertRegexpMatches(). - body = notice.get_payload() - self.assertIn('test@example.com', body) - self.assertIn('anne@example.com', body) - self.assertIn('http://example.com/anne@example.com', body) - self.assertIn('test-owner@example.com', body) + # interpolation values appear in the message. + self.assertMultiLineEqual(notice.get_payload(), """\ +This is a probe message. You can ignore this message. + +The test@example.com mailing list has received a number of bounces +from you, indicating that there may be a problem delivering messages +to anne@example.com. A sample is attached below. Please examine this +message to make sure there are no problems with your email address. +You may want to check with your mail administrator for more help. + +You don't need to do anything to remain an enabled member of the +mailing list. + +If you have any questions or problems, you can contact the mailing +list owner at + + test-owner@example.com +""") def test_headers(self): # Check the headers of the outer message. @@ -285,7 +297,8 @@ Message-ID: self._var_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self._var_dir) xx_template_path = os.path.join( - self._var_dir, 'templates', 'site', 'xx', 'probe.txt') + self._var_dir, 'templates', 'site', 'xx', + 'list:user:notice:probe.txt') os.makedirs(os.path.dirname(xx_template_path)) config.push('xx template dir', """\ [paths.testing] @@ -300,7 +313,6 @@ Message-ID: blah blah blah $listname $address -$optionsurl $owneraddr """, file=fp) @@ -322,7 +334,9 @@ $owneraddr notice = message.get_payload(0).get_payload() self.assertMultiLineEqual(notice, """\ blah blah blah test@example.com anne@example.com -http://example.com/anne@example.com test-owner@example.com""") +test-owner@example.com + +""") class TestProbe(unittest.TestCase): @@ -364,6 +378,7 @@ class TestMaybeForward(unittest.TestCase): """Test forwarding of unrecognized bounces.""" layer = ConfigLayer + maxDiff = None def setUp(self): config.push('test config', """ @@ -425,9 +440,12 @@ Message-ID: payload = msg.get_payload(0) self.assertEqual(payload.get_content_type(), 'text/plain') body = payload.get_payload() - self.assertEqual( - body.splitlines()[-1], - 'http://lists.example.com/admin/test@example.com/bounce') + self.assertMultiLineEqual(body, """\ +The attached message was received as a bounce, but either the bounce format +was not recognized, or no member addresses could be extracted from it. This +mailing list has been configured to send all unrecognized bounce messages to +the list administrator(s). +""") # The second attachment should be a message/rfc822 containing the # original bounce message. payload = msg.get_payload(1) @@ -470,9 +488,12 @@ Message-ID: payload = msg.get_payload(0) self.assertEqual(payload.get_content_type(), 'text/plain') body = payload.get_payload() - self.assertEqual( - body.splitlines()[-1], - 'http://lists.example.com/admin/test@example.com/bounce') + self.assertMultiLineEqual(body, """\ +The attached message was received as a bounce, but either the bounce format +was not recognized, or no member addresses could be extracted from it. This +mailing list has been configured to send all unrecognized bounce messages to +the list administrator(s). +""") # The second attachment should be a message/rfc822 containing the # original bounce message. payload = msg.get_payload(1) diff --git a/src/mailman/app/tests/test_notifications.py b/src/mailman/app/tests/test_notifications.py index 09873f9bb..3230fd892 100644 --- a/src/mailman/app/tests/test_notifications.py +++ b/src/mailman/app/tests/test_notifications.py @@ -18,19 +18,20 @@ """Test notifications.""" import os -import shutil -import tempfile import unittest +from contextlib import ExitStack from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.member import MemberRole +from mailman.interfaces.template import ITemplateManager from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( get_queue_messages, set_preferred, subscribe) from mailman.testing.layers import ConfigLayer from mailman.utilities.datetime import now +from tempfile import TemporaryDirectory from zope.component import getUtility @@ -41,36 +42,37 @@ class TestNotifications(unittest.TestCase): maxDiff = None def setUp(self): + resources = ExitStack() + self.addCleanup(resources.close) + self.var_dir = resources.enter_context(TemporaryDirectory()) self._mlist = create_list('test@example.com') - self._mlist.welcome_message_uri = 'mailman:///welcome.txt' self._mlist.display_name = 'Test List' - self.var_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.var_dir) + getUtility(ITemplateManager).set( + 'user:ack:welcome', self._mlist.list_id, 'mailman:///welcome.txt') config.push('template config', """\ [paths.testing] template_dir: {}/templates """.format(self.var_dir)) - self.addCleanup(config.pop, 'template config') + resources.callback(config.pop, 'template config') # Populate the template directories with a few fake templates. path = os.path.join(self.var_dir, 'templates', 'site', 'en') os.makedirs(path) - with open(os.path.join(path, 'welcome.txt'), 'w') as fp: + full_path = os.path.join(path, 'list:user:notice:welcome.txt') + with open(full_path, 'w', encoding='utf-8') as fp: print("""\ Welcome to the $list_name mailing list. Posting address: $fqdn_listname Help and other requests: $list_requests Your name: $user_name - Your address: $user_address - Your options: $user_options_uri""", file=fp) + Your address: $user_address""", file=fp) # Write a list-specific welcome message. path = os.path.join(self.var_dir, 'templates', 'lists', 'test@example.com', 'xx') os.makedirs(path) - with open(os.path.join(path, 'welcome.txt'), 'w') as fp: + full_path = os.path.join(path, 'list:user:notice:welcome.txt') + with open(full_path, 'w', encoding='utf-8') as fp: print('You just joined the $list_name mailing list!', file=fp) - # Let assertMultiLineEqual work without bounds. - self.maxDiff = None def test_welcome_message(self): subscribe(self._mlist, 'Anne', email='anne@example.com') @@ -86,13 +88,13 @@ Welcome to the Test List mailing list. Help and other requests: test-request@example.com Your name: Anne Person Your address: anne@example.com - Your options: http://example.com/anne@example.com """) def test_more_specific_welcome_message_nonenglish(self): - # mlist.welcome_message_uri can contain placeholders for the fqdn list + # The welcome message url can contain placeholders for the fqdn list # name and language. - self._mlist.welcome_message_uri = ( + getUtility(ITemplateManager).set( + 'user:ack:welcome', self._mlist.list_id, 'mailman:///$listname/$language/welcome.txt') # Add the xx language and subscribe Anne using it. manager = getUtility(ILanguageManager) diff --git a/src/mailman/app/tests/test_registrar.py b/src/mailman/app/tests/test_registrar.py index cec90c88b..2107f9648 100644 --- a/src/mailman/app/tests/test_registrar.py +++ b/src/mailman/app/tests/test_registrar.py @@ -247,7 +247,8 @@ class TestRegistrar(unittest.TestCase): self.assertEqual(message['To'], 'ant-owner@example.com') self.assertEqual(message['Subject'], 'Ant subscription notification') self.assertEqual(message.get_payload(), """\ -anne@example.com has been successfully subscribed to Ant.""") +anne@example.com has been successfully subscribed to Ant. +""") def test_no_admin_notify_mchanges(self): # Even when a user gets subscribed via the subscription policy diff --git a/src/mailman/app/tests/test_subscriptions.py b/src/mailman/app/tests/test_subscriptions.py index 11512bcfe..9f02593a9 100644 --- a/src/mailman/app/tests/test_subscriptions.py +++ b/src/mailman/app/tests/test_subscriptions.py @@ -450,7 +450,8 @@ Your authorization is required for a mailing list subscription request approval: For: anne@example.com - List: test@example.com""") + List: test@example.com +""") def test_get_moderator_approval_no_notifications(self): # When the subscription is held for moderator approval, and the list diff --git a/src/mailman/app/tests/test_templates.py b/src/mailman/app/tests/test_templates.py deleted file mode 100644 index 59bf74a0b..000000000 --- a/src/mailman/app/tests/test_templates.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (C) 2012-2016 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Test the template downloader API.""" - -import os -import shutil -import tempfile -import unittest - -from mailman.app.lifecycle import create_list -from mailman.config import config -from mailman.interfaces.templates import ITemplateLoader -from mailman.testing.layers import ConfigLayer -from urllib.error import URLError -from zope.component import getUtility - - -class TestTemplateLoader(unittest.TestCase): - """Test the template downloader API.""" - - layer = ConfigLayer - - def setUp(self): - self.var_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.var_dir) - config.push('template config', """\ - [paths.testing] - var_dir: {0} - """.format(self.var_dir)) - self.addCleanup(config.pop, 'template config') - # Put a demo template in the site directory. - path = os.path.join(self.var_dir, 'templates', 'site', 'en') - os.makedirs(path) - with open(os.path.join(path, 'demo.txt'), 'w') as fp: - print('Test content', end='', file=fp) - self._loader = getUtility(ITemplateLoader) - self._mlist = create_list('test@example.com') - - def test_mailman_internal_uris(self): - # mailman://demo.txt - content = self._loader.get('mailman:///demo.txt') - self.assertEqual(content, 'Test content') - - def test_mailman_internal_uris_twice(self): - # mailman:///demo.txt - content = self._loader.get('mailman:///demo.txt') - self.assertEqual(content, 'Test content') - content = self._loader.get('mailman:///demo.txt') - self.assertEqual(content, 'Test content') - - def test_mailman_uri_with_language(self): - content = self._loader.get('mailman:///en/demo.txt') - self.assertEqual(content, 'Test content') - - def test_mailman_uri_with_english_fallback(self): - content = self._loader.get('mailman:///it/demo.txt') - self.assertEqual(content, 'Test content') - - def test_mailman_uri_with_list_name(self): - content = self._loader.get('mailman:///test@example.com/demo.txt') - self.assertEqual(content, 'Test content') - - def test_mailman_full_uri(self): - content = self._loader.get('mailman:///test@example.com/en/demo.txt') - self.assertEqual(content, 'Test content') - - def test_mailman_full_uri_with_english_fallback(self): - content = self._loader.get('mailman:///test@example.com/it/demo.txt') - self.assertEqual(content, 'Test content') - - def test_uri_not_found(self): - with self.assertRaises(URLError) as cm: - self._loader.get('mailman:///missing.txt') - self.assertEqual(cm.exception.reason, 'No such file') - - def test_shorter_url_error(self): - with self.assertRaises(URLError) as cm: - self._loader.get('mailman:///') - self.assertEqual(cm.exception.reason, 'No template specified') - - def test_short_url_error(self): - with self.assertRaises(URLError) as cm: - self._loader.get('mailman://') - self.assertEqual(cm.exception.reason, 'No template specified') - - def test_bad_language(self): - with self.assertRaises(URLError) as cm: - self._loader.get('mailman:///xx/demo.txt') - self.assertEqual(cm.exception.reason, 'Bad language or list name') - - def test_bad_mailing_list(self): - with self.assertRaises(URLError) as cm: - self._loader.get('mailman:///missing@example.com/demo.txt') - self.assertEqual(cm.exception.reason, 'Bad language or list name') - - def test_too_many_path_components(self): - with self.assertRaises(URLError) as cm: - self._loader.get('mailman:///missing@example.com/en/foo/demo.txt') - self.assertEqual(cm.exception.reason, 'No such file') - - def test_non_ascii(self): - # mailman://demo.txt with non-ascii content. - test_text = b'\xe4\xb8\xad' - path = os.path.join(self.var_dir, 'templates', 'site', 'it') - os.makedirs(path) - with open(os.path.join(path, 'demo.txt'), 'wb') as fp: - fp.write(test_text) - content = self._loader.get('mailman:///it/demo.txt') - self.assertIsInstance(content, str) - self.assertEqual(content, test_text.decode('utf-8')) diff --git a/src/mailman/archiving/docs/common.rst b/src/mailman/archiving/docs/common.rst index 5666e9a39..8670cf98a 100644 --- a/src/mailman/archiving/docs/common.rst +++ b/src/mailman/archiving/docs/common.rst @@ -38,8 +38,8 @@ interoperate. http://go.mail-archive.dev/test%40example.com http://go.mail-archive.dev/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE mhonarc - http://lists.example.com/.../test@example.com - http://lists.example.com/.../RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE + http://example.com/.../test@example.com + http://example.com/.../RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE prototype None None diff --git a/src/mailman/archiving/mhonarc.py b/src/mailman/archiving/mhonarc.py index 3aa482bff..857a61e06 100644 --- a/src/mailman/archiving/mhonarc.py +++ b/src/mailman/archiving/mhonarc.py @@ -50,11 +50,11 @@ class MHonArc: def list_url(self, mlist): """See `IArchiver`.""" # XXX What about private MHonArc archives? - return expand(self.base_url, - dict(listname=mlist.fqdn_listname, - hostname=mlist.domain.url_host, - fqdn_listname=mlist.fqdn_listname, - )) + return expand(self.base_url, mlist, dict( + # For backward compatibility. + hostname=mlist.domain.mail_host, + fqdn_listname=mlist.fqdn_listname, + )) def permalink(self, mlist, msg): """See `IArchiver`.""" @@ -77,7 +77,7 @@ class MHonArc: """See `IArchiver`.""" substitutions = config.__dict__.copy() substitutions['listname'] = mlist.fqdn_listname - command = expand(self.command, substitutions) + command = expand(self.command, mlist, substitutions) proc = Popen( command, stdin=PIPE, stdout=PIPE, stderr=PIPE, diff --git a/src/mailman/chains/hold.py b/src/mailman/chains/hold.py index 3a8284a78..f1bd42961 100644 --- a/src/mailman/chains/hold.py +++ b/src/mailman/chains/hold.py @@ -33,9 +33,9 @@ from mailman.interfaces.autorespond import IAutoResponseSet, Response from mailman.interfaces.chain import HoldEvent from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.pending import IPendable, IPendings +from mailman.interfaces.template import ITemplateLoader from mailman.interfaces.usermanager import IUserManager -from mailman.utilities.i18n import make -from mailman.utilities.string import oneline, wrap +from mailman.utilities.string import expand, oneline, wrap from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -100,13 +100,17 @@ def autorespond_to_sender(mlist, sender, language=None): log.info('hold autoresponse limit hit: %s', sender) response_set.response_sent(address, Response.hold) # Send this notification message instead. - text = make('nomoretoday.txt', - language=language.code, - sender=sender, - listname=mlist.fqdn_listname, - count=todays_count, - owneremail=mlist.owner_address, - ) + template = getUtility(ITemplateLoader).get( + 'list:user:notice:no-more-today', mlist, + language=language.code) + text = wrap(expand(template, mlist, dict( + language=language.code, + count=todays_count, + sender_email=sender, + # For backward compatibility. + sender=sender, + owneremail=mlist.owner_address, + ))) with _.using(language.code): msg = UserNotification( sender, mlist.owner_address, @@ -163,10 +167,11 @@ class HoldChain(TerminalChainBase): bytes_subject = oneline_subject.encode(charset, 'replace') original_subject = bytes_subject.decode(charset) substitutions = dict( - listname = mlist.fqdn_listname, # noqa - subject = original_subject, # noqa - sender = msg.sender, # noqa - reasons = _compose_reasons(msgdata), # noqa + subject=original_subject, + sender_email=msg.sender, + reasons=_compose_reasons(msgdata), + # For backward compatibility. + sender=msg.sender, ) # At this point the message is held, but now we have to craft at least # two responses. The first will go to the original author of the @@ -191,10 +196,12 @@ class HoldChain(TerminalChainBase): subject = _( 'Your message to $mlist.fqdn_listname awaits moderator approval') send_language_code = msgdata.get('lang', language.code) - text = make('postheld.txt', - mailing_list=mlist, - language=send_language_code, - **substitutions) + template = getUtility(ITemplateLoader).get( + 'list:user:notice:hold', mlist, + language=send_language_code) + text = wrap(expand(template, mlist, dict( + language=send_language_code, + **substitutions))) adminaddr = mlist.bounces_address nmsg = UserNotification( msg.sender, adminaddr, subject, text, @@ -221,10 +228,9 @@ class HoldChain(TerminalChainBase): mlist.owner_address, subject, lang=language) nmsg.set_type('multipart/mixed') - text = MIMEText(make('postauth.txt', - mailing_list=mlist, - wrap=False, - **substitutions), + template = getUtility(ITemplateLoader).get( + 'list:admin:action:post', mlist) + text = MIMEText(expand(template, mlist, substitutions), _charset=charset) dmsg = MIMEText(wrap(_("""\ If you reply to this message, keeping the Subject: header intact, Mailman will diff --git a/src/mailman/commands/cli_lists.py b/src/mailman/commands/cli_lists.py index 352accab2..9813da29c 100644 --- a/src/mailman/commands/cli_lists.py +++ b/src/mailman/commands/cli_lists.py @@ -30,7 +30,8 @@ from mailman.interfaces.domain import ( BadDomainSpecificationError, IDomainManager) from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager, ListAlreadyExistsError -from mailman.utilities.i18n import make +from mailman.interfaces.template import ITemplateLoader +from mailman.utilities.string import expand, wrap from zope.component import getUtility from zope.interface import implementer @@ -221,14 +222,13 @@ class Create: if not args.quiet: print(_('Created mailing list: $mlist.fqdn_listname')) if args.notify: - d = dict( - listname = mlist.fqdn_listname, # noqa - admin_url = mlist.script_url('admin'), # noqa - listinfo_url = mlist.script_url('listinfo'), # noqa - requestaddr = mlist.request_address, # noqa - siteowner = mlist.no_reply_address, # noqa - ) - text = make('newlist.txt', mailing_list=mlist, **d) + template = getUtility(ITemplateLoader).get( + 'domain:admin:notice:new-list', mlist) + text = wrap(expand(template, mlist, dict( + # For backward compatibility. + requestaddr=mlist.request_address, + siteowner=mlist.no_reply_address, + ))) # Set the I18N language to the list's preferred language so the # header will match the template language. Stashing and restoring # the old translation context is just (healthy? :) paranoia. diff --git a/src/mailman/commands/docs/create.rst b/src/mailman/commands/docs/create.rst index 701a51a08..12e3a1c95 100644 --- a/src/mailman/commands/docs/create.rst +++ b/src/mailman/commands/docs/create.rst @@ -44,7 +44,7 @@ Now both the domain and the mailing list exist in the database. >>> from mailman.interfaces.domain import IDomainManager >>> getUtility(IDomainManager).get('example.xx') - + You can prevent the creation of the domain in existing domains by using the ``-D`` or ``--no-domain`` flag. Although the ``--no-domain`` flag is not @@ -169,17 +169,9 @@ The notification message is in the virgin queue. The mailing list 'test6@example.com' has just been created for you. The following is some basic information about your mailing list. - You can configure your mailing list at the following web page: - - http://lists.example.com/admin/test6@example.com - - The web page for users of your mailing list is: - - http://lists.example.com/listinfo/test6@example.com - - There is also an email-based interface for users (not administrators) - of your list; you can get info about using it by sending a message - with just the word 'help' as subject or in the body, to: + There is an email-based interface for users (not administrators) of + your list; you can get info about using it by sending a message with + just the word 'help' as subject or in the body, to: test6-request@example.com diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst index 407b9eacc..5cea9ee2c 100644 --- a/src/mailman/commands/docs/info.rst +++ b/src/mailman/commands/docs/info.rst @@ -64,6 +64,7 @@ definition. File system paths: ARCHIVE_DIR = /var/lib/mailman/archives BIN_DIR = /sbin + CACHE_DIR = /var/lib/mailman/cache CFG_FILE = .../test.cfg DATA_DIR = /var/lib/mailman/data ETC_DIR = /etc diff --git a/src/mailman/commands/tests/test_lists.py b/src/mailman/commands/tests/test_lists.py index 392e3f36e..4d0b0e14a 100644 --- a/src/mailman/commands/tests/test_lists.py +++ b/src/mailman/commands/tests/test_lists.py @@ -42,8 +42,7 @@ class TestLists(unittest.TestCase): def test_lists_with_domain_option(self): # LP: #1166911 - non-matching lists were returned. getUtility(IDomainManager).add( - 'example.net', 'An example domain.', - 'http://lists.example.net') + 'example.net', 'An example domain.') create_list('test1@example.com') create_list('test2@example.com') # Only this one should show up. diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 415e1e3d2..5454c5e5a 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -156,8 +156,8 @@ class Configuration: else category.template_dir), ) # Directories. - for name in ('archive', 'bin', 'data', 'etc', 'ext', 'list_data', - 'lock', 'log', 'messages', 'queue'): + for name in ('archive', 'bin', 'cache', 'data', 'etc', 'ext', + 'list_data', 'lock', 'log', 'messages', 'queue'): key = '{}_dir'.format(name) substitutions[key] = getattr(category, key) # Files. diff --git a/src/mailman/config/configure.zcml b/src/mailman/config/configure.zcml index 535cf729f..f31e79525 100644 --- a/src/mailman/config/configure.zcml +++ b/src/mailman/config/configure.zcml @@ -57,6 +57,11 @@ factory="mailman.model.bounce.BounceProcessor" /> + + + + _url placeholder for every enabled archiver. for archiver in IListArchiverSet(mlist).archivers: @@ -71,20 +72,10 @@ def process(mlist, msg, msgdata): d[placeholder] = archive_url # These strings are descriptive for the log file and shouldn't be i18n'd d.update(msgdata.get('decoration-data', {})) - try: - header = decorate(mlist, mlist.header_uri, d) - except URLError: - header = None - log.exception('Header decorator URI not found ({0}): {1}'.format( - mlist.fqdn_listname, mlist.header_uri)) - try: - footer = decorate(mlist, mlist.footer_uri, d) - except URLError: - footer = None - log.exception('Footer decorator URI not found ({0}): {1}'.format( - mlist.fqdn_listname, mlist.footer_uri)) + header = decorate('list:member:regular:header', mlist, d) + footer = decorate('list:member:regular:footer', mlist, d) # Escape hatch if both the footer and header are empty or None. - if not header and not footer: + if len(header) == 0 and len(footer) == 0: return # Be MIME smart here. We only attach the header and footer by # concatenation when the message is a non-multipart of type text/plain. @@ -120,9 +111,9 @@ def process(mlist, msg, msgdata): oldpayload = msg.get_payload(decode=True).decode(mcset) del msg['content-transfer-encoding'] frontsep = endsep = '' - if header and not header.endswith('\n'): + if len(header) > 0 and not header.endswith('\n'): frontsep = '\n' - if footer and not oldpayload.endswith('\n'): + if len(footer) > 0 and not oldpayload.endswith('\n'): endsep = '\n' payload = header + frontsep + oldpayload + endsep + footer # When setting the payload for the message, try various charset @@ -152,11 +143,11 @@ def process(mlist, msg, msgdata): payload = msg.get_payload() if not isinstance(payload, list): payload = [payload] - if footer: + if len(footer) > 0: mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) mimeftr['Content-Disposition'] = 'inline' payload.append(mimeftr) - if header: + if len(header) > 0: mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) mimehdr['Content-Disposition'] = 'inline' payload.insert(0, mimehdr) @@ -194,11 +185,11 @@ def process(mlist, msg, msgdata): # subparts: the header (if any), the wrapped message, and the footer (if # any). payload = [inner] - if header: + if len(header) > 0: mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) mimehdr['Content-Disposition'] = 'inline' payload.insert(0, mimehdr) - if footer: + if len(footer) > 0: mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) mimeftr['Content-Disposition'] = 'inline' payload.append(mimeftr) @@ -210,18 +201,12 @@ def process(mlist, msg, msgdata): @public -def decorate(mlist, uri, extradict=None): - """Expand the decoration template from its URI.""" - if uri is None: - return '' +def decorate(name, mlist, extradict=None): + """Expand the named decoration template uri.""" + if extradict is None: + extradict = {} # Get the decorator template. - loader = getUtility(ITemplateLoader) - template_uri = expand(uri, dict( - language=mlist.preferred_language.code, - list_id=mlist.list_id, - listname=mlist.fqdn_listname, - )) - template = loader.get(template_uri) + template = getUtility(ITemplateLoader).get(name, mlist, **extradict) return decorate_template(mlist, template, extradict) @@ -242,11 +227,9 @@ def decorate_template(mlist, template, extradict=None): 'info', ) } - # This must eventually go away. - substitutions['listinfo_uri'] = mlist.script_url('listinfo') if extradict is not None: substitutions.update(extradict) - text = expand(template, substitutions) + text = expand(template, mlist, substitutions) # Turn any \r\n line endings into just \n return re.sub(r' *\r?\n', r'\n', text) diff --git a/src/mailman/handlers/docs/acknowledge.rst b/src/mailman/handlers/docs/acknowledge.rst index 42cab04a0..fbe46b063 100644 --- a/src/mailman/handlers/docs/acknowledge.rst +++ b/src/mailman/handlers/docs/acknowledge.rst @@ -133,10 +133,6 @@ The receipt will include the original message's subject in the response body, Something witty and insightful was successfully received by the Test mailing list. - - List info page: http://lists.example.com/listinfo/test@example.com - Your preferences: http://example.com/aperson@example.com - If there is no subject, then the receipt will use a generic message. @@ -169,7 +165,3 @@ If there is no subject, then the receipt will use a generic message. (no subject) was successfully received by the Test mailing list. - - List info page: http://lists.example.com/listinfo/test@example.com - Your preferences: http://example.com/aperson@example.com - diff --git a/src/mailman/handlers/docs/decorate.rst b/src/mailman/handlers/docs/decorate.rst index e6199f8e0..7e02ea9eb 100644 --- a/src/mailman/handlers/docs/decorate.rst +++ b/src/mailman/handlers/docs/decorate.rst @@ -6,7 +6,7 @@ Message decoration is the process of adding headers and footers to the original message. A handler module takes care of this based on the settings of the mailing list and the type of message being processed. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('ant@example.com') >>> msg_text = """\ ... From: aperson@example.org ... @@ -41,7 +41,7 @@ Simple decorations Message decorations are specified by URI and can be specialized by the mailing list and language. Internal Mailman decorations can be referenced by using -the ``mailman://`` URL scheme. Here we create a simple English header and +the ``mailman:///`` URL scheme. Here we create a simple English header and footer for all mailing lists in our site. :: @@ -51,7 +51,7 @@ footer for all mailing lists in our site. >>> os.makedirs(site_dir) >>> config.push('templates', """ ... [paths.testing] - ... template_dir: {0} + ... template_dir: {} ... """.format(template_dir)) >>> myheader_path = os.path.join(site_dir, 'myheader.txt') @@ -61,11 +61,17 @@ footer for all mailing lists in our site. >>> with open(myfooter_path, 'w') as fp: ... print('footer', file=fp) -Setting these attributes on the mailing list causes it to use these -templates. Since these are site-global templates, we can use a shorter path. +Adding these template URIs to the template manager sets the mailing list up to +use these templates. Since these are site-global templates, we can use a +shorter path. - >>> mlist.header_uri = 'mailman:///myheader.txt' - >>> mlist.footer_uri = 'mailman:///myfooter.txt' + >>> from mailman.interfaces.template import ITemplateManager + >>> from zope.component import getUtility + >>> manager = getUtility(ITemplateManager) + >>> manager.set('list:member:regular:header', + ... mlist.list_id, 'mailman:///myheader.txt') + >>> manager.set('list:member:regular:footer', + ... mlist.list_id, 'mailman:///myfooter.txt') Text messages that have no declared content type are, by default encoded in ASCII. When the mailing list's preferred language is ``en`` (i.e. English), @@ -94,14 +100,14 @@ short descriptive name for the mailing list). ... print('$display_name footer', file=fp) >>> msg = message_from_string(msg_text) - >>> mlist.display_name = 'XTest' + >>> mlist.display_name = 'Ant' >>> process(mlist, msg, {}) >>> print(msg.as_string()) From: aperson@example.org ... - XTest header + Ant header Here is a message. - XTest footer + Ant footer You can't just pick any interpolation variable though; if you do, the variable will remain in the header or footer unchanged. diff --git a/src/mailman/handlers/docs/rfc-2369.rst b/src/mailman/handlers/docs/rfc-2369.rst index 0f3cb5c5e..e2d2d6d47 100644 --- a/src/mailman/handlers/docs/rfc-2369.rst +++ b/src/mailman/handlers/docs/rfc-2369.rst @@ -73,10 +73,8 @@ have a reduced set of `List-` headers. Specifically, there is no `List-Post`, ---start--- list-help: list-id: - list-subscribe: , - - list-unsubscribe: , - + list-subscribe: + list-unsubscribe: ---end--- @@ -98,10 +96,8 @@ header which contains the `mailto:` URL used to send messages to the list. list-help: list-id: list-post: - list-subscribe: , - - list-unsubscribe: , - + list-subscribe: + list-unsubscribe: ---end--- Some mailing lists are announce, or one-way lists, not discussion lists. @@ -120,10 +116,8 @@ to RFC 2369. list-help: list-id: list-post: NO - list-subscribe: , - - list-unsubscribe: , - + list-subscribe: + list-unsubscribe: ---end--- @@ -145,10 +139,8 @@ header. list-help: list-id: My test mailing list list-post: - list-subscribe: , - - list-unsubscribe: , - + list-subscribe: + list-unsubscribe: ---end--- Any existing ``List-Id`` headers are removed from the original message. diff --git a/src/mailman/handlers/replybot.py b/src/mailman/handlers/replybot.py index 546bb1ec4..a86ccbbf3 100644 --- a/src/mailman/handlers/replybot.py +++ b/src/mailman/handlers/replybot.py @@ -100,12 +100,11 @@ class Replybot: d = dict( list_name=mlist.list_name, display_name=display_name, - listurl=mlist.script_url('listinfo'), requestemail=mlist.request_address, owneremail=mlist.owner_address, ) # Interpolation and Wrap the response text. - text = wrap(expand(response_text, d)) + text = wrap(expand(response_text, mlist, d)) outmsg = UserNotification(msg.sender, mlist.bounces_address, subject, text, mlist.preferred_language) outmsg['X-Mailer'] = _('The Mailman Replybot') diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py index 59c618da8..e042bc5e3 100644 --- a/src/mailman/handlers/rfc_2369.py +++ b/src/mailman/handlers/rfc_2369.py @@ -57,8 +57,6 @@ def process(mlist, msg, msgdata): # "X-List-Administrivia: yes" header. For all others (i.e. those coming # from list posts), we add a bunch of other RFC 2369 headers. requestaddr = mlist.request_address - subfieldfmt = '<{}>, ' - listinfo = mlist.script_url('listinfo') headers = [] # XXX reduced_list_headers used to suppress List-Help, List-Subject, and # List-Unsubscribe from UserNotification. That doesn't seem to make sense @@ -66,9 +64,8 @@ def process(mlist, msg, msgdata): # suppressed). headers.extend(( ('List-Help', ''.format(requestaddr)), - ('List-Unsubscribe', - subfieldfmt.format(listinfo, mlist.leave_address)), - ('List-Subscribe', subfieldfmt.format(listinfo, mlist.join_address)), + ('List-Unsubscribe', ''.format(mlist.leave_address)), + ('List-Subscribe', ''.format(mlist.join_address)), )) if not msgdata.get('reduced_list_headers'): # List-Post: is controlled by a separate attribute, which is somewhat diff --git a/src/mailman/handlers/tests/test_decorate.py b/src/mailman/handlers/tests/test_decorate.py index 7d44214cb..2187a908b 100644 --- a/src/mailman/handlers/tests/test_decorate.py +++ b/src/mailman/handlers/tests/test_decorate.py @@ -24,10 +24,12 @@ from mailman.app.lifecycle import create_list from mailman.config import config from mailman.handlers import decorate from mailman.interfaces.archiver import IArchiver +from mailman.interfaces.template import ITemplateManager from mailman.testing.helpers import ( LogFileMark, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer from tempfile import TemporaryDirectory +from zope.component import getUtility from zope.interface import implementer @@ -86,7 +88,8 @@ This is a test message. footer_path = os.path.join(site_dir, 'myfooter.txt') with open(footer_path, 'w', encoding='utf-8') as fp: print('${testarchiver_url}', file=fp) - self._mlist.footer_uri = 'mailman:///myfooter.txt' + getUtility(ITemplateManager).set( + 'list:member:regular:footer', None, 'mailman:///myfooter.txt') self._mlist.preferred_language = 'en' decorate.process(self._mlist, self._msg, {}) self.assertIn('http://example.com/link_to_message', @@ -100,7 +103,9 @@ This is a test message. footer_path = os.path.join(list_dir, 'myfooter.txt') with open(footer_path, 'w', encoding='utf-8') as fp: print('${testarchiver_url}', file=fp) - self._mlist.footer_uri = 'mailman:///${list_id}/myfooter.txt' + getUtility(ITemplateManager).set( + 'list:member:regular:footer', self._mlist.list_id, + 'mailman:///${list_id}/myfooter.txt') self._mlist.preferred_language = 'en' decorate.process(self._mlist, self._msg, {}) self.assertIn('http://example.com/link_to_message', @@ -114,7 +119,8 @@ This is a test message. footer_path = os.path.join(list_dir, 'myfooter.txt') with open(footer_path, 'w', encoding='utf-8') as fp: print('${testarchiver_url}', file=fp) - self._mlist.footer_uri = ( + getUtility(ITemplateManager).set( + 'list:member:regular:footer', self._mlist.list_id, 'mailman:///${list_id}/${language}/myfooter.txt') self._mlist.preferred_language = 'it' decorate.process(self._mlist, self._msg, {}) @@ -155,7 +161,9 @@ This is a test message. footer_path = os.path.join(site_dir, 'myfooter.txt') with open(footer_path, 'w', encoding='utf-8') as fp: print('${broken_url}', file=fp) - self._mlist.footer_uri = 'mailman:///myfooter.txt' + getUtility(ITemplateManager).set( + 'list:member:regular:footer', self._mlist.list_id, + 'mailman:///myfooter.txt') self._mlist.preferred_language = 'en' mark = LogFileMark('mailman.archiver') decorate.process(self._mlist, self._msg, {}) diff --git a/src/mailman/handlers/tests/test_rfc_2369.py b/src/mailman/handlers/tests/test_rfc_2369.py index e78cf9c47..41b94ae0f 100644 --- a/src/mailman/handlers/tests/test_rfc_2369.py +++ b/src/mailman/handlers/tests/test_rfc_2369.py @@ -37,7 +37,7 @@ class DummyArchiver: def list_url(self, mlist): """See `IArchiver`.""" - return mlist.domain.base_url + return 'http://{}'.format(mlist.mail_host) def permalink(self, mlist, msg): """See `IArchiver`.""" @@ -105,11 +105,10 @@ Dummy text self.addCleanup(config.pop, 'archiver') rfc_2369.process(self._mlist, self._msg, {}) self.assertEqual( - self._msg.get_all('List-Archive'), - ['']) + self._msg.get_all('List-Archive'), ['']) self.assertEqual( self._msg.get_all('Archived-At'), - ['']) + ['']) def test_prototype_no_url(self): # The prototype archiver is not web-based, it must not return URLs diff --git a/src/mailman/interfaces/cache.py b/src/mailman/interfaces/cache.py new file mode 100644 index 000000000..74fb2084a --- /dev/null +++ b/src/mailman/interfaces/cache.py @@ -0,0 +1,66 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""File caches.""" + +from mailman import public +from zope.interface import Interface + + +@public +class ICacheManager(Interface): + """Manager for managing cached files.""" + + def add(key, contents, lifetime=None): + """Add the contents to the cache, indexed by the key. + + If there is already some contents cached under the given key, the old + contents are overwritten with the new contents. + + :param key: The key to use when storing the contents. + :type name: str + :param contents: The contents to store in the cache. If this is a + bytes object, it will be stored on disk in binary. If it's a str, + it will be stored in UTF-8. Either way, the manager will remember + the type and return it when you access the file. + :type contents: bytes or str + :param lifetime: How long should the file be cached for, before it + expires (leading to its eventual eviction)? If not given, the + system default lifetime is used. + :type lifetime: datetime.timedelta + :return: The SHA256 hash under which the file contents are stored. + :rtype: str + """ + + def get(key, *, expunge=False): + """Return the contents cached under the given key. + + :param key: The key identifying the contents you want to retrieve. + :type key: str + :param expunge: A flag indicating whether the file contents should + also be removed from the cache. + :type expunge: bool + :return: The contents of the cached file or None if the given id isn't + in the cache (or it's already expired). + :rtype: bytes or str, depending on the original contents. + """ + + def evict(): + """Evict all files which have expired.""" + + def clear(): + """Clear the entire cache of files.""" diff --git a/src/mailman/interfaces/domain.py b/src/mailman/interfaces/domain.py index 7be5909a9..b1940f400 100644 --- a/src/mailman/interfaces/domain.py +++ b/src/mailman/interfaces/domain.py @@ -69,16 +69,6 @@ class IDomain(Interface): mail_host = Attribute('The host name for email for this domain.') - url_host = Attribute( - 'The host name for the web interface for this domain.') - - base_url = Attribute("""\ - The base url for the Mailman server at this domain, which includes the - scheme and host name.""") - - scheme = Attribute( - """The protocol scheme used to contact this list's server.""") - description = Attribute( 'The human readable description of the domain name.') @@ -91,31 +81,18 @@ class IDomain(Interface): The mailing lists are returned in order sorted by list-id. """) - def confirm_url(token=''): - """The url used for various forms of confirmation. - - :param token: The confirmation token to use in the url. - :type token: string - :return: The confirmation url. - :rtype: string - """ - @public class IDomainManager(Interface): """The manager of domains.""" - def add(mail_host, description=None, base_url=None, owners=None): + def add(mail_host, description=None, owners=None): """Add a new domain. :param mail_host: The email host name for the domain. :type mail_host: string :param description: The description of the domain. :type description: string - :param base_url: The base url, including the scheme for the web - interface of the domain. If not given, it defaults to - http://`mail_host`/ - :type base_url: string :param owners: Sequence of owners of the domain, defaults to None, meaning the domain does not have owners. :type owners: sequence of `IUser` or string emails. diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index b5c0e5456..19db025d1 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -111,6 +111,19 @@ class IMailingList(Interface): mailing lists, or in headers, and so forth. It should be as succinct as you can get it, while still identifying what the list is.""") + info = Attribute("""\ + A longer description of this mailing list. This can be any arbitrary + text, up to a database-specific maximum length. + """) + + preferred_language = Attribute("""\ + The default language for communications on this mailing list. + + When the list sends out notifications, it will be in this language, + unless the recipient is a known user and that user has a preferred + language. + """) + subject_prefix = Attribute("""\ The text to insert at the front of the Subject field. @@ -349,29 +362,6 @@ class IMailingList(Interface): digest recipients are cleared. """) - # Web access. - - scheme = Attribute( - """The protocol scheme used to contact this list's server. - - The web server on this protocol provides the web interface for this - mailing list. The protocol scheme should be 'http' or 'https'.""") - - web_host = Attribute( - """This list's web server's domain. - - The read-only domain name of the host to contact for interacting with - the web interface of the mailing list.""") - - def script_url(target, context=None): - """Return the url to the given script target. - - If 'context' is not given, or is None, then an absolute url is - returned. If context is given, it must be an IMailingListRequest - object, and the returned url will be relative to that object's - 'location' attribute. - """ - # Autoresponses. autoresponse_grace_period = Attribute( @@ -593,183 +583,9 @@ class IMailingList(Interface): send_welcome_message = Attribute( """Flag indicating whether a welcome message should be sent.""") - welcome_message_uri = Attribute( - """URI for the list's welcome message. - - This can be any URI supported by `urllib2` with the addition of - `mailman:` URIs, which reference internal default resources. This is - a template which can include the following placeholders: - - $listname - the FQDN list name for this mailing list. - $language - the language code, usually the list's preferred language. - - The resource will be downloaded and cached whenever the welcome - message is sent. The resource at this URI can contain the following - placeholders, which are also filled in through values on the mailing - list: - - $fqdn_listname - the FQDN list name for this mailing list. - $list_name - the human readable name for the mailing list. - $listinfo_uri - the URI to the list's information page. - $list_requests - the address to the list's `-request` address. - $user_name - the name of the subscribing user. - $user_address - the email address of the subscribing user. - $user_options_uri - the URI to this member's options page. - """) - send_goodbye_message = Attribute( """Flag indicating whether a goodbye message should be sent.""") - goodbye_message_uri = Attribute( - """URI for the list's goodbye message. - - This can be any URI supported by `urllib2` with the addition of - `mailman:` URIs, which reference internal default resources. This is - a template which can include the following placeholders: - - $listname - the FQDN list name for this mailing list. - $language - the language code, usually the list's preferred language. - - The resource will be downloaded and cached whenever the goodbye - message is sent. The resource at this URI can contain the following - placeholders, which are also filled in through values on the mailing - list: - - $fqdn_listname - the FQDN list name for this mailing list. - $list_name - the human readable name for the mailing list. - $listinfo_uri - the URI to the list's information page. - $list_requests - the address to the list's `-request` address. - $user_name - the name of the subscribing user. - $user_address - the email address of the subscribing user. - $user_options_uri - the URI to this member's options page. - """) - - # Decorators. - - header_uri = Attribute( - """URI for the header decorator on regular delivery messages. - - This can be any URI supported by `urllib2` with the addition of - `mailman:` URIs, which reference internal default resources. This is - a template which can include the following placeholders: - - $listname - the FQDN list name for this mailing list. - $language - the language code, usually the list's preferred language. - - The resource will be downloaded and cached whenever the decorator is - needed. The resource at this URI can contain the following - placeholders, which are also filled in through values on the mailing - list: - - $fqdn_listname - the FQDN list name for this mailing list. - $list_name - the human readable name for the mailing list. - $host_name - the mailing list's host name - $listinfo_uri - the URI to the list's information page. - $list_requests - the address to the list's `-request` address. - $description - the mailing list's description - $info - additional mailing list's information - - Personalized messages will also have these placeholders available: - - $user_name - the name of the subscribing user. - $user_address - the email address of the subscribing user. - $user_options_uri - the URI to this member's options page. - """ - ) - - footer_uri = Attribute( - """URI for the footer decorator on regular delivery messages. - - This can be any URI supported by `urllib2` with the addition of - `mailman:` URIs, which reference internal default resources. This is - a template which can include the following placeholders: - - $listname - the FQDN list name for this mailing list. - $language - the language code, usually the list's preferred language. - - The resource will be downloaded and cached whenever the decorator is - needed. The resource at this URI can contain the following - placeholders, which are also filled in through values on the mailing - list: - - $fqdn_listname - the FQDN list name for this mailing list. - $list_name - the human readable name for the mailing list. - $host_name - the mailing list's host name - $listinfo_uri - the URI to the list's information page. - $list_requests - the address to the list's `-request` address. - $description - the mailing list's description - $info - additional mailing list's information - - Personalized messages will also have these placeholders available: - - $user_name - the name of the subscribing user. - $user_address - the email address of the subscribing user. - $user_options_uri - the URI to this member's options page. - """ - ) - - digest_header_uri = Attribute( - """URI for the header decorator on digest messages. - - This can be any URI supported by `urllib2` with the addition of - `mailman:` URIs, which reference internal default resources. This is - a template which can include the following placeholders: - - $listname - the FQDN list name for this mailing list. - $language - the language code, usually the list's preferred language. - - The resource will be downloaded and cached whenever the decorator is - needed. The resource at this URI can contain the following - placeholders, which are also filled in through values on the mailing - list: - - $fqdn_listname - the FQDN list name for this mailing list. - $list_name - the human readable name for the mailing list. - $host_name - the mailing list's host name - $listinfo_uri - the URI to the list's information page. - $list_requests - the address to the list's `-request` address. - $description - the mailing list's description - $info - additional mailing list's information - - Personalized messages will also have these placeholders available: - - $user_name - the name of the subscribing user. - $user_address - the email address of the subscribing user. - $user_options_uri - the URI to this member's options page. - """ - ) - - digest_footer_uri = Attribute( - """URI for the footer decorator on digest messages. - - This can be any URI supported by `urllib2` with the addition of - `mailman:` URIs, which reference internal default resources. This is - a template which can include the following placeholders: - - $listname - the FQDN list name for this mailing list. - $language - the language code, usually the list's preferred language. - - The resource will be downloaded and cached whenever the decorator is - needed. The resource at this URI can contain the following - placeholders, which are also filled in through values on the mailing - list: - - $fqdn_listname - the FQDN list name for this mailing list. - $list_name - the human readable name for the mailing list. - $host_name - the mailing list's host name - $listinfo_uri - the URI to the list's information page. - $list_requests - the address to the list's `-request` address. - $description - the mailing list's description - $info - additional mailing list's information - - Personalized messages will also have these placeholders available: - - $user_name - the name of the subscribing user. - $user_address - the email address of the subscribing user. - $user_options_uri - the URI to this member's options page. - """ - ) - @public class IAcceptableAlias(Interface): diff --git a/src/mailman/interfaces/member.py b/src/mailman/interfaces/member.py index 367838e21..094652521 100644 --- a/src/mailman/interfaces/member.py +++ b/src/mailman/interfaces/member.py @@ -174,6 +174,16 @@ class IMember(Interface): address, it will be an ``IUser``. """) + display_name = Attribute( + """The best match of the member's display name. + + This will be `subscriber.display_name` if available, which means it + will either be the display name of the address or user that's + subscribed. If unavailable, and the address is the subscriber, then + the linked user's display name is given, if available. When all else + fails, the empty string is returned. + """) + preferences = Attribute( """This member's preferences.""") @@ -258,11 +268,3 @@ class IMember(Interface): 4. System default XXX I'm not sure this is the right place to put this.""") - - options_url = Attribute( - """Return the url for the given member's option page. - - XXX This needs a serious re-think in the face of the unified user - database, since a member's options aren't tied to any specific mailing - list. So in what part of the web-space does the user's options live? - """) diff --git a/src/mailman/interfaces/template.py b/src/mailman/interfaces/template.py new file mode 100644 index 000000000..442f1eeda --- /dev/null +++ b/src/mailman/interfaces/template.py @@ -0,0 +1,199 @@ +# Copyright (C) 2012-2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Template downloader with cache.""" + +from mailman import public +from zope.interface import Attribute, Interface + + +@public +class ITemplate(Interface): + """A template record.""" + + name = Attribute("""The template name.""") + + context = Attribute("""\ + The template context. This may be a list-id, a domain mail host name, + or None (for the global context). + """) + + uri = Attribute("""The template uri.""") + + username = Attribute("""\ + Optional username used to retrieved the contents referenced by the + template uri. This can be None if no username/password is needed.""") + + password = Attribute("""\ + Optional password used to retrieved the contents referenced by the + template uri. This can be None if no username/password is needed.""") + + +@public +class ITemplateLoader(Interface): + """The template downloader utility.""" + + def get(name, context=None, **kws): + """Find the named template for the given context. + + This search for a named template using a stacked strategy. If the + template has been registered for the given context in the template + manager, that template is used. If not, a file system based template + is used as a fallback. If nothing can be found, the empty string is + returned. + + Use the ``ITemplateManager`` to set specific template locations. The + fallbacks all use the ``mailman:`` scheme to use in-tree defaults. + + :param name: The name of the template to find. + :type name: str + :param context: The context in which to find the template. This can + be either an IMailingList, an IDomain, or None for the global/site + context. The template will be searched in order from most + specific to least specific, i.e. from list-id, to domain, to + global. + :type context: IMailingList, IDomain, None + :param kws: Additional URL substitution variables. Once a URL for the + given name and context is identified, these are used to fill in + placeholders in the URL before the template is retrieved. + `list_id`, `list_name`, and `mail_host` will automatically be + filled in if available depending on the `context`. This + dictionary is passed directly to the underlying + ``ITemplatemanager.get()`` call. + :type kws: dict + :return: The found template or its fallback. + :rtype: str + """ + + +@public +class ITemplateManager(Interface): + """Manager/loader for notification templates.""" + + def set(name, context, uri, username=None, password=''): + """Set a template mapping from name to URI. + + The URI may be cached for some length of time defined by the system. + + :param name: The template name, including any extension. + :type name: str + :param context: The context for this name->URI mapping. This can be a + list-id, domain mail host name, or None (for global context). + :type context: str + :param uri: The URI of the template. Normal http: and https: URIs can + be used, as well as special mailman: URIs which reference internal + resources. + :type uri: str + :param username: Optional user name for Basic Auth on the URI. + :type username: str + :param password: Optional password for Basic Auth on the URI. + :type username: str + """ + + def get(name, context, **kws): + """Return the contents for the given context and name. + + `context` can be a list-id, domain mail host name, or "*" (for global + context). A search will be performed from the named context up to the + global context. For example, if a List-Id is given but no template + for that mailing list under that name is registered, the mailing + list's domain is search, then the global context is searched. + + If the URI mapped to this mailing list/name pair is not yet retrieved, + it is downloaded first. If the cache lifetime has expired, it will be + downloaded again. Otherwise the cached version will be returned. + + :param name: The template name, including any extension. + :type name: str + :param context: The context for this name->URI mapping. This can be a + List-ID, domain mail host name, or None (for global context). + :type context: str + :param kws: A substitution dictionary that is interpolated into the + url to retrieve the contents of the template. Passing in a + different dictionary than before can cause a new template to be + downloaded. + :type kws: dict + :return: The resource mapped to the given name, or None if not found. + :rtype: str or None + """ + + def raw(name, context): + """Return the raw template information for the given context and name. + + `context` can be a list-id, domain mail host name, or "*" (for global + context). The raw template matching the given name and context is + returned, otherwise None if no such template has been registered. + + :param name: The template name, including any extension. + :type name: str + :param context: The context for this name->URI mapping. This can be a + List-ID, domain mail host name, or None (for global context). + :type context: str + :return: The raw template record, or None if not found. + :rtype: ITemplate or None + """ + + def delete(name, context): + """Delete the named template and any cached contents. + + :param name: The template name, including any extension. + :type name: str + :param context: The context for this name->URI mapping. This can be a + List-ID, domain mail host name, or None (for global context). + :type context: str + """ + +# Mapping of template names to their in-source file names. A None value means +# that there is no file in the tree for that template. + +ALL_TEMPLATES = { + key: '{}.txt'.format(key) + for key in { + 'domain:admin:notice:new-list', + 'list:admin:action:post', + 'list:admin:action:subscribe', + 'list:admin:action:unsubscribe', + 'list:admin:notice:subscribe', + 'list:admin:notice:unrecognized', + 'list:admin:notice:unsubscribe', + 'list:member:digest:masthead', + 'list:user:action:confirm', + 'list:user:action:unsubscribe', + 'list:user:notice:hold', + 'list:user:notice:no-more-today', + 'list:user:notice:post', + 'list:user:notice:probe', + 'list:user:notice:refuse', + 'list:user:notice:welcome', + } + } + +# These have other names. +ALL_TEMPLATES.update({ + 'list:member:digest:footer': 'list:member:generic:footer.txt', + 'list:member:regular:footer': 'list:member:generic:footer.txt', + }) + +# These are some extra supported templates which don't have a mapping to a +# file in the source tree. +ALL_TEMPLATES.update({ + 'list:member:digest:header': None, + 'list:member:regular:header': None, + 'list:user:notice:goodbye': None, + }) + +public(ALL_TEMPLATES=ALL_TEMPLATES) diff --git a/src/mailman/interfaces/templates.py b/src/mailman/interfaces/templates.py deleted file mode 100644 index 40bc86d2a..000000000 --- a/src/mailman/interfaces/templates.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2012-2016 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Template downloader with cache.""" - -from mailman import public -from zope.interface import Interface - - -@public -class ITemplateLoader(Interface): - """The template downloader utility.""" - - def get(uri): - """Download the named URI, and return the response and content. - - This API uses `urllib2`_ so consult its documentation for details. - - .. _`urllib2`: http://docs.python.org/library/urllib2.html - - :param uri: The URI of the resource. These may be any URI supported - by `urllib2` and also `mailman:` URIs for internal resources. - :type uri: string - :return: The template string as a unicode. - :rtype: str - """ diff --git a/src/mailman/model/cache.py b/src/mailman/model/cache.py new file mode 100644 index 000000000..d9c0d0445 --- /dev/null +++ b/src/mailman/model/cache.py @@ -0,0 +1,160 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Generic file cache.""" + +import os +import hashlib + +from contextlib import ExitStack +from lazr.config import as_timedelta +from mailman import public +from mailman.config import config +from mailman.database.model import Model +from mailman.database.transaction import dbconnection +from mailman.interfaces.cache import ICacheManager +from mailman.utilities.datetime import now +from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode +from zope.interface import implementer + + +class CacheEntry(Model): + __tablename__ = 'file_cache' + + id = Column(Integer, primary_key=True) + key = Column(Unicode) + file_id = Column(Unicode) + is_bytes = Column(Boolean) + created_on = Column(DateTime) + expires_on = Column(DateTime) + + @dbconnection + def __init__(self, store, key, file_id, is_bytes, lifetime): + self.key = key + self.file_id = file_id + self.is_bytes = is_bytes + self.created_on = now() + self.expires_on = self.created_on + lifetime + + @dbconnection + def update(self, store, is_bytes, lifetime): + self.is_bytes = is_bytes + self.created_on = now() + self.expires_on = self.created_on + lifetime + + @property + def is_expired(self): + return self.expires_on <= now() + + +@public +@implementer(ICacheManager) +class CacheManager: + """Manages a cache of files on the file system.""" + + @staticmethod + def _id_to_path(file_id): + dir_1 = file_id[0:2] + dir_2 = file_id[2:4] + dir_path = os.path.join(config.CACHE_DIR, dir_1, dir_2) + file_path = os.path.join(dir_path, file_id) + return file_path, dir_path + + @staticmethod + def _key_to_file_id(key): + # Calculate the file-id/SHA256 hash. The key must be a string, even + # though the hash algorithm requires bytes. + hashfood = key.encode('raw-unicode-escape') + # Use the hex digest (a str) for readability. + return hashlib.sha256(hashfood).hexdigest() + + def _write_contents(self, file_id, contents, is_bytes): + # Calculate the file system path by taking the SHA1 hash, stripping + # out two levels of directory (to reduce the chance of direntry + # exhaustion on some systems). + file_path, dir_path = self._id_to_path(file_id) + os.makedirs(dir_path, exist_ok=True) + # Open the file on the correct mode and write the contents. + with ExitStack() as resources: + if is_bytes: + fp = resources.enter_context(open(file_path, 'wb')) + else: + fp = resources.enter_context( + open(file_path, 'w', encoding='utf-8')) + fp.write(contents) + + @dbconnection + def add(self, store, key, contents, lifetime=None): + """See `ICacheManager`.""" + if lifetime is None: + lifetime = as_timedelta(config.mailman.cache_life) + is_bytes = isinstance(contents, bytes) + file_id = self._key_to_file_id(key) + # Is there already an unexpired entry under this id in the database? + # If the entry doesn't exist, create it. If it overwrite both the + # contents and lifetime. + entry = store.query(CacheEntry).filter( + CacheEntry.key == key).one_or_none() + if entry is None: + entry = CacheEntry(key, file_id, is_bytes, lifetime) + store.add(entry) + else: + entry.update(is_bytes, lifetime) + self._write_contents(file_id, contents, is_bytes) + return file_id + + @dbconnection + def get(self, store, key, *, expunge=False): + """See `ICacheManager`.""" + entry = store.query(CacheEntry).filter( + CacheEntry.key == key).one_or_none() + if entry is None: + return None + file_path, dir_path = self._id_to_path(entry.file_id) + with ExitStack() as resources: + if entry.is_bytes: + fp = resources.enter_context(open(file_path, 'rb')) + else: + fp = resources.enter_context( + open(file_path, 'r', encoding='utf-8')) + contents = fp.read() + # Do we expunge the cache file? + if expunge: + store.delete(entry) + os.remove(file_path) + return contents + + @dbconnection + def evict(self, store): + """See `ICacheManager`.""" + # Find all the cache entries which have expired. We can probably do + # this more efficiently, but for now there probably aren't that many + # cached files. + for entry in store.query(CacheEntry): + if entry.is_expired: + file_path, dir_path = self._id_to_path(entry.file_id) + os.remove(file_path) + store.delete(entry) + + @dbconnection + def clear(self, store): + # Delete all the entries. We can probably do this more efficiently, + # but for now there probably aren't that many cached files. + for entry in store.query(CacheEntry): + file_path, dir_path = self._id_to_path(entry.file_id) + os.remove(file_path) + store.delete(entry) diff --git a/src/mailman/model/docs/domains.rst b/src/mailman/model/docs/domains.rst index c0d65cee7..dc117a98e 100644 --- a/src/mailman/model/docs/domains.rst +++ b/src/mailman/model/docs/domains.rst @@ -32,24 +32,24 @@ Adding a domain requires some basic information, of which the email host name is the only required piece. The other parts are inferred from that. >>> manager.add('example.org') - + >>> show_domains() - + We can remove domains too. >>> manager.remove('example.org') - + >>> show_domains() no domains Sometimes the email host name is different than the base url for hitting the web interface for the domain. - >>> manager.add('example.com', base_url='https://mail.example.com') - + >>> manager.add('example.com') + >>> show_domains() - + Domains can have explicit descriptions, and can be created with one or more owners. @@ -57,16 +57,13 @@ owners. >>> manager.add( ... 'example.net', - ... base_url='http://lists.example.net', ... description='The example domain', ... owners=['anne@example.com']) - + >>> show_domains(with_owners=True) - - + + - owner: anne@example.com Domains can have multiple owners, ideally one of the owners should have a @@ -76,8 +73,8 @@ configuration's default contact address may be used as a fallback. >>> net_domain = manager['example.net'] >>> net_domain.add_owner('bart@example.org') >>> show_domains(with_owners=True) - - + + - owner: anne@example.com - owner: bart@example.org @@ -114,30 +111,17 @@ In the global domain manager, domains are indexed by their email host name. example.net >>> print(manager['example.net']) - + As with dictionaries, you can also get the domain. If the domain does not exist, ``None`` or a default is returned. :: >>> print(manager.get('example.net')) - + >>> print(manager.get('doesnotexist.com')) None >>> print(manager.get('doesnotexist.com', 'blahdeblah')) blahdeblah - - -Confirmation tokens -=================== - -Confirmation tokens can be added to the domain's url to generate the URL to a -page users can use to confirm their subscriptions. - - >>> domain = manager['example.net'] - >>> print(domain.confirm_url('abc')) - http://lists.example.net/confirm/abc diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 5eb67fa95..575d157cf 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -28,7 +28,6 @@ from mailman.interfaces.usermanager import IUserManager from mailman.model.mailinglist import MailingList from sqlalchemy import Column, Integer, Unicode from sqlalchemy.orm import relationship -from urllib.parse import urljoin, urlparse from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -44,7 +43,6 @@ class Domain(Model): id = Column(Integer, primary_key=True) mail_host = Column(Unicode) - base_url = Column(Unicode) description = Column(Unicode) owners = relationship('User', secondary='domain_owner', @@ -52,7 +50,6 @@ class Domain(Model): def __init__(self, mail_host, description=None, - base_url=None, owners=None): """Create and register a domain. @@ -60,31 +57,14 @@ class Domain(Model): :type mail_host: string :param description: An optional description of the domain. :type description: string - :param base_url: The optional base url for the domain, including - scheme. If not given, it will be constructed from the - `mail_host` using the http protocol. - :type base_url: string :param owners: Optional owners of this domain. :type owners: sequence of `IUser` or string emails. """ self.mail_host = mail_host - self.base_url = (base_url - if base_url is not None - else 'http://' + mail_host) self.description = description if owners is not None: self.add_owners(owners) - @property - def url_host(self): - """See `IDomain`.""" - return urlparse(self.base_url).netloc - - @property - def scheme(self): - """See `IDomain`.""" - return urlparse(self.base_url).scheme - @property @dbconnection def mailing_lists(self, store): @@ -93,18 +73,12 @@ class Domain(Model): MailingList.mail_host == self.mail_host ).order_by(MailingList._list_id) - def confirm_url(self, token=''): - """See `IDomain`.""" - return urljoin(self.base_url, 'confirm/' + token) - def __repr__(self): """repr(a_domain)""" if self.description is None: - return ('').format( - self) + return ('').format(self) else: - return ('').format(self) + return ('').format(self) def add_owner(self, owner): """See `IDomain`.""" @@ -140,7 +114,6 @@ class DomainManager: def add(self, store, mail_host, description=None, - base_url=None, owners=None): """See `IDomainManager`.""" # Be sure the mail_host is not already registered. This is probably @@ -149,7 +122,7 @@ class DomainManager: raise BadDomainSpecificationError( 'Duplicate email host: {}'.format(mail_host)) notify(DomainCreatingEvent(mail_host)) - domain = Domain(mail_host, description, base_url, owners) + domain = Domain(mail_host, description, owners) store.add(domain) notify(DomainCreatedEvent(domain)) return domain diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index b147acd72..9d7d63d9b 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -56,7 +56,6 @@ from sqlalchemy.event import listen from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from sqlalchemy.orm.exc import NoResultFound -from urllib.parse import urljoin from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -133,8 +132,6 @@ class MailingList(Model): default_nonmember_action = Column(Enum(Action)) description = Column(Unicode) digests_enabled = Column(Boolean) - digest_footer_uri = Column(Unicode) - digest_header_uri = Column(Unicode) digest_is_default = Column(Boolean) digest_send_periodic = Column(Boolean) digest_size_threshold = Column(Float) @@ -143,12 +140,9 @@ class MailingList(Model): emergency = Column(Boolean) encode_ascii_prefixes = Column(Boolean) first_strip_reply_to = Column(Boolean) - footer_uri = Column(Unicode) forward_auto_discards = Column(Boolean) gateway_to_mail = Column(Boolean) gateway_to_news = Column(Boolean) - goodbye_message_uri = Column(Unicode) - header_uri = Column(Unicode) hold_these_nonmembers = Column(PickleType) info = Column(Unicode) linked_newsgroup = Column(Unicode) @@ -184,7 +178,6 @@ class MailingList(Model): topics = Column(PickleType) topics_bodylines_limit = Column(Integer) topics_enabled = Column(Boolean) - welcome_message_uri = Column(Unicode) # ORM relationships. header_matches = relationship( 'HeaderMatch', backref='mailing_list', @@ -244,22 +237,6 @@ class MailingList(Model): """See `IMailingList`.""" return getUtility(IDomainManager)[self.mail_host] - @property - def scheme(self): - """See `IMailingList`.""" - return self.domain.scheme - - @property - def web_host(self): - """See `IMailingList`.""" - return self.domain.url_host - - def script_url(self, target, context=None): - """See `IMailingList`.""" - # XXX Handle the case for when context is not None; those would be - # relative URLs. - return urljoin(self.domain.base_url, target + '/' + self.fqdn_listname) - @property def data_path(self): """See `IMailingList`.""" @@ -314,7 +291,7 @@ class MailingList(Model): def confirm_address(self, cookie): """See `IMailingList`.""" - local_part = expand(config.mta.verp_confirm_format, dict( + local_part = expand(config.mta.verp_confirm_format, self, dict( address='{}-confirm'.format(self.list_name), cookie=cookie)) return '{}@{}'.format(local_part, self.mail_host) diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py index 26c36b305..3aaa5e392 100644 --- a/src/mailman/model/member.py +++ b/src/mailman/model/member.py @@ -131,6 +131,21 @@ class Member(Model): def subscriber(self): return (self._user if self._address is None else self._address) + @property + def display_name(self): + # Try to find a non-empty display name. We first look at the directly + # subscribed record, which will either be the address or the user. + # That's handled automatically by going through member.subscriber. If + # that doesn't give us something useful, try whatever user is linked + # to the subscriber. + if self.subscriber.display_name: + return self.subscriber.display_name + # If an unlinked address is subscribed tehre will be no .user. + elif self.user is not None and self.user.display_name: + return self.user.display_name + else: + return '' + def _lookup(self, preference, default=None): pref = getattr(self.preferences, preference) if pref is not None: @@ -180,12 +195,6 @@ class Member(Model): """See `IMember`.""" return self._lookup('delivery_status') - @property - def options_url(self): - """See `IMember`.""" - # XXX Um, this is definitely wrong - return 'http://example.com/' + self.address.email - @dbconnection def unsubscribe(self, store): """See `IMember`.""" diff --git a/src/mailman/model/template.py b/src/mailman/model/template.py new file mode 100644 index 000000000..96ce43e1f --- /dev/null +++ b/src/mailman/model/template.py @@ -0,0 +1,202 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Template management.""" + +import logging + +from mailman import public +from mailman.config import config +from mailman.database.model import Model +from mailman.database.transaction import dbconnection +from mailman.interfaces.cache import ICacheManager +from mailman.interfaces.domain import IDomain +from mailman.interfaces.mailinglist import IMailingList +from mailman.interfaces.template import ( + ALL_TEMPLATES, ITemplateLoader, ITemplateManager) +from mailman.utilities import protocols +from mailman.utilities.i18n import find +from mailman.utilities.string import expand +from requests import HTTPError +from sqlalchemy import Column, Integer, Unicode +from urllib.error import URLError +from urllib.parse import urlparse +from zope.component import getUtility +from zope.interface import implementer + + +COMMASPACE = ', ' +log = logging.getLogger('mailman.http') + + +class Template(Model): + __tablename__ = 'template' + + id = Column(Integer, primary_key=True) + name = Column(Unicode) + context = Column(Unicode) + uri = Column(Unicode) + username = Column(Unicode, nullable=True) + password = Column(Unicode, nullable=True) + + def __init__(self, name, context, uri, username, password): + self.name = name + self.context = context + self.reset(uri, username, password) + + def reset(self, uri, username, password): + self.uri = uri + self.username = username + self.password = password + + +@public +@implementer(ITemplateManager) +class TemplateManager: + """Manager of templates, with caching and support for mailman:// URIs.""" + + @dbconnection + def set(self, store, name, context, uri, username=None, password=''): + """See `ITemplateManager`.""" + # Just record the fact that we have a template set. Make sure that if + # there is an existing template with the same context and name, we + # override any of its settings (and evict the cache). + template = store.query(Template).filter( + Template.name == name, + Template.context == context).one_or_none() + if template is None: + template = Template(name, context, uri, username, password) + store.add(template) + else: + template.reset(uri, username, password) + + @dbconnection + def get(self, store, name, context, **kws): + """See `ITemplateManager`.""" + template = store.query(Template).filter( + Template.name == name, + Template.context == context).one_or_none() + if template is None: + return None + actual_uri = expand(template.uri, None, kws) + cache_mgr = getUtility(ICacheManager) + contents = cache_mgr.get(actual_uri) + if contents is None: + # It's likely that the cached contents have expired. + auth = {} + if template.username is not None: + auth['auth'] = (template.username, template.password) + try: + contents = protocols.get(actual_uri, **auth) + except HTTPError as error: + # 404/NotFound errors are interpreted as missing templates, + # for which we'll return the default (i.e. the empty string). + # All other exceptions get passed up the chain. + if error.response.status_code != 404: + raise + log.exception('Cannot retrieve template at {} ({})'.format( + actual_uri, auth.get('auth', ''))) + return '' + # We don't need to cache mailman: contents since those are already + # on the file system. + if urlparse(actual_uri).scheme != 'mailman': + cache_mgr.add(actual_uri, contents) + return contents + + @dbconnection + def raw(self, store, name, context): + """See `ITemplateManager`.""" + return store.query(Template).filter( + Template.name == name, + Template.context == context).one_or_none() + + @dbconnection + def delete(self, store, name, context): + """See `ITemplateManager`.""" + template = store.query(Template).filter( + Template.name == name, + Template.context == context).one_or_none() + if template is not None: + store.delete(template) + # We don't clear the cache entry, we just let it expire. + + +@public +@implementer(ITemplateLoader) +class TemplateLoader: + """Loader of templates.""" + + def get(self, name, context=None, **kws): + """See `ITemplateLoader`.""" + # Gather some additional information based on the context. + substitutions = {} + if IMailingList.providedBy(context): + mlist = context + domain = context.domain + lookup_contexts = [ + mlist.list_id, + mlist.mail_host, + None, + ] + substitutions.update(dict( + list_id=mlist.list_id, + # For backward compatibility, we call this $listname. + listname=mlist.fqdn_listname, + domain_name=domain.mail_host, + language=mlist.preferred_language.code, + )) + elif IDomain.providedBy(context): + mlist = None + domain = context + lookup_contexts = [ + domain.mail_host, + None, + ] + substitutions['domain_name'] = domain.mail_host + elif context is None: + mlist = domain = None + lookup_contexts = [None] + else: + raise ValueError('Bad context type: {!r}'.format(context)) + # The passed in keyword arguments take precedence. + substitutions.update(kws) + # See if there's a cached template registered for this name and + # context, passing in the url substitutions. This handles http:, + # https:, and file: urls. + for lookup_context in lookup_contexts: + try: + contents = getUtility(ITemplateManager).get( + name, lookup_context, **substitutions) + except (HTTPError, URLError): + pass + else: + if contents is not None: + return contents + # Fallback to searching within the source code. + code = substitutions.get('language', config.mailman.default_language) + # Find the template, mutating any missing template exception. + missing = object() + default_uri = ALL_TEMPLATES.get(name, missing) + if default_uri is None: + return '' + elif default_uri is missing: + raise URLError('No such file') + path, fp = find(default_uri, mlist, code) + try: + return fp.read() + finally: + fp.close() diff --git a/src/mailman/model/tests/test_cache.py b/src/mailman/model/tests/test_cache.py new file mode 100644 index 000000000..2d910e202 --- /dev/null +++ b/src/mailman/model/tests/test_cache.py @@ -0,0 +1,110 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the cache.""" + +import os +import unittest + +from datetime import timedelta +from mailman.config import config +from mailman.interfaces.cache import ICacheManager +from mailman.testing.helpers import configuration +from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import factory +from zope.component import getUtility + + +class TestCache(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._cachemgr = getUtility(ICacheManager) + + def test_add_str_contents(self): + file_id = self._cachemgr.add('abc', 'xyz') + self.assertEqual( + file_id, + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad') + file_path = os.path.join(config.CACHE_DIR, 'ba', '78', file_id) + self.assertTrue(os.path.exists(file_path)) + # The original content was a string. + with open(file_path, 'r', encoding='utf-8') as fp: + self.assertEqual(fp.read(), 'xyz') + + def test_add_bytes_contents(self): + # No name is given so the file is cached by the hash of the contents. + file_id = self._cachemgr.add('abc', b'xyz') + self.assertEqual( + file_id, + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad') + file_path = os.path.join(config.CACHE_DIR, 'ba', '78', file_id) + self.assertTrue(os.path.exists(file_path)) + # The original content was a string. + with open(file_path, 'br') as fp: + self.assertEqual(fp.read(), b'xyz') + + def test_add_overwrite(self): + # If the file already exists and hasn't expired, a conflict exception + # is raised the second time we try to save it. + self._cachemgr.add('abc', 'xyz') + self.assertEqual(self._cachemgr.get('abc'), 'xyz') + self._cachemgr.add('abc', 'def') + self.assertEqual(self._cachemgr.get('abc'), 'def') + + def test_get_str(self): + # Store a str, get a str. + self._cachemgr.add('abc', 'xyz') + contents = self._cachemgr.get('abc') + self.assertEqual(contents, 'xyz') + + def test_get_bytes(self): + # Store a bytes, get a bytes. + self._cachemgr.add('abc', b'xyz') + contents = self._cachemgr.get('abc') + self.assertEqual(contents, b'xyz') + + def test_get_str_expunge(self): + # When the entry is not expunged, it can be gotten multiple times. + # Once it's expunged, it's gone. + self._cachemgr.add('abc', 'xyz') + self.assertEqual(self._cachemgr.get('abc'), 'xyz') + self.assertEqual(self._cachemgr.get('abc', expunge=True), 'xyz') + self.assertIsNone(self._cachemgr.get('abc')) + + @configuration('mailman', cache_life='1d') + def test_evict(self): + # Evicting all expired cache entries makes them inaccessible. + self._cachemgr.add('abc', 'xyz', lifetime=timedelta(hours=3)) + self._cachemgr.add('def', 'uvw', lifetime=timedelta(days=3)) + self.assertEqual(self._cachemgr.get('abc'), 'xyz') + self.assertEqual(self._cachemgr.get('def'), 'uvw') + factory.fast_forward(days=1) + self._cachemgr.evict() + self.assertIsNone(self._cachemgr.get('abc')) + self.assertEqual(self._cachemgr.get('def'), 'uvw') + + def test_clear(self): + # Clearing the cache gets rid of all entries, regardless of lifetime. + self._cachemgr.add('abc', 'xyz', lifetime=timedelta(hours=3)) + self._cachemgr.add('def', 'uvw') + self.assertEqual(self._cachemgr.get('abc'), 'xyz') + self.assertEqual(self._cachemgr.get('def'), 'uvw') + factory.fast_forward(days=1) + self._cachemgr.clear() + self.assertIsNone(self._cachemgr.get('abc')) + self.assertIsNone(self._cachemgr.get('xyz')) diff --git a/src/mailman/model/tests/test_template.py b/src/mailman/model/tests/test_template.py new file mode 100644 index 000000000..f16a4aa5c --- /dev/null +++ b/src/mailman/model/tests/test_template.py @@ -0,0 +1,286 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the template manager.""" + +import unittest +import threading + +from contextlib import ExitStack +from http.server import BaseHTTPRequestHandler, HTTPServer +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.domain import IDomainManager +from mailman.interfaces.template import ITemplateLoader, ITemplateManager +from mailman.testing.helpers import wait_for_webservice +from mailman.testing.layers import ConfigLayer +from mailman.utilities.i18n import find +from requests import HTTPError +from tempfile import TemporaryDirectory +from urllib.error import URLError +from zope.component import getUtility + +# New in Python 3.5. +try: + from http import HTTPStatus +except ImportError: + class HTTPStatus: + FORBIDDEN = 403 + NOT_FOUND = 404 + OK = 200 + + +# We need a web server to vend non-mailman: urls. +class TestableHandler(BaseHTTPRequestHandler): + # Be quiet. + def log_request(*args, **kws): + pass + + log_error = log_request + + def do_GET(self): + if self.path == '/welcome_3.txt': + if self.headers['Authorization'] != 'Basic YW5uZTppcyBzcGVjaWFs': + self.send_error(HTTPStatus.FORBIDDEN) + return + response = TEXTS.get(self.path) + if response is None: + self.send_error(HTTPStatus.NOT_FOUND) + return + self.send_response(HTTPStatus.OK) + self.send_header('Content-Type', 'UTF-8') + self.end_headers() + self.wfile.write(response.encode('utf-8')) + + +class HTTPLayer(ConfigLayer): + httpd = None + + @classmethod + def setUp(cls): + assert cls.httpd is None, 'Layer already set up' + cls.httpd = HTTPServer(('localhost', 8180), TestableHandler) + cls._thread = threading.Thread(target=cls.httpd.serve_forever) + cls._thread.daemon = True + cls._thread.start() + wait_for_webservice('localhost', 8180) + + @classmethod + def tearDown(cls): + assert cls.httpd is not None, 'Layer not set up' + cls.httpd.shutdown() + cls.httpd.server_close() + cls._thread.join() + + +class TestTemplateCache(unittest.TestCase): + layer = HTTPLayer + + def setUp(self): + self._templatemgr = getUtility(ITemplateManager) + + def test_http_set_get(self): + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_1.txt') + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com') + self.assertEqual(contents, WELCOME_1) + + def test_http_set_override(self): + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_1.txt') + # Resetting the template with the same context and domain, but a + # different url overrides the previous value. + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_2.txt') + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com') + self.assertEqual(contents, WELCOME_2) + + def test_http_get_cached(self): + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_1.txt') + # The first one warms the cache. + self._templatemgr.get('list:user:notice:welcome', 'test.example.com') + # The second one hits the cache. + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com') + self.assertEqual(contents, WELCOME_1) + + def test_http_basic_auth(self): + # We get an HTTP error when we forget the username and password. + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_3.txt') + with self.assertRaises(HTTPError) as cm: + self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com') + self.assertEqual(cm.exception.response.status_code, 403) + self.assertEqual(cm.exception.response.reason, 'Forbidden') + # But providing the basic auth information let's it work. + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_3.txt', + username='anne', password='is special') + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com') + self.assertEqual(contents, WELCOME_3) + + def test_delete(self): + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/welcome_1.txt') + self._templatemgr.get('list:user:notice:welcome', 'test.example.com') + self._templatemgr.delete( + 'list:user:notice:welcome', 'test.example.com') + self.assertIsNone( + self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com')) + + def test_delete_missing(self): + self._templatemgr.delete( + 'list:user:notice:welcome', 'test.example.com') + self.assertIsNone( + self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com')) + + def test_get_keywords(self): + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/${path}_${number}.txt') + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com', + path='welcome', number='1') + self.assertEqual(contents, WELCOME_1) + + def test_get_different_keywords(self): + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/${path}_${number}.txt') + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com', + path='welcome', number='1') + self.assertEqual(contents, WELCOME_1) + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com', + path='welcome', number='2') + self.assertEqual(contents, WELCOME_2) + + def test_not_found(self): + # A 404 is treated specially, resulting in the empty string. + self._templatemgr.set( + 'list:user:notice:welcome', 'test.example.com', + 'http://localhost:8180/missing.txt') + contents = self._templatemgr.get( + 'list:user:notice:welcome', 'test.example.com') + self.assertEqual(contents, '') + + +class TestTemplateLoader(unittest.TestCase): + """Test the template downloader API.""" + + layer = HTTPLayer + + def setUp(self): + resources = ExitStack() + self.addCleanup(resources.close) + var_dir = resources.enter_context(TemporaryDirectory()) + config.push('template config', """\ + [paths.testing] + var_dir: {} + """.format(var_dir)) + resources.callback(config.pop, 'template config') + self._mlist = create_list('test@example.com') + self._loader = getUtility(ITemplateLoader) + self._manager = getUtility(ITemplateManager) + + def test_domain_context(self): + self._manager.set( + 'list:user:notice:welcome', 'example.com', + 'http://localhost:8180/$domain_name/welcome_4.txt') + domain = getUtility(IDomainManager).get('example.com') + content = self._loader.get('list:user:notice:welcome', domain) + self.assertEqual(content, 'This is a domain welcome.\n') + + def test_domain_content_fallback(self): + self._manager.set( + 'list:user:notice:welcome', 'example.com', + 'http://localhost:8180/$domain_name/welcome_4.txt') + content = self._loader.get('list:user:notice:welcome', self._mlist) + self.assertEqual(content, 'This is a domain welcome.\n') + + def test_site_context(self): + self._manager.set( + 'list:user:notice:welcome', None, + 'http://localhost:8180/welcome_2.txt') + content = self._loader.get('list:user:notice:welcome') + self.assertEqual(content, "Sure, I guess you're welcome.\n") + + def test_site_context_mailman(self): + self._manager.set( + 'list:user:notice:welcome', None, + 'mailman:///welcome.txt') + template_content = self._loader.get('list:user:notice:welcome') + path, fp = find('list:user:notice:welcome.txt') + try: + found_contents = fp.read() + finally: + fp.close() + self.assertEqual(template_content, found_contents) + + def test_bad_context(self): + self.assertRaises( + ValueError, self._loader.get, 'list:user:notice:welcome', object()) + + def test_no_such_file(self): + self.assertRaises(URLError, self._loader.get, 'missing', self._mlist) + + def test_403_forbidden(self): + # 404s are swallowed, but not 403s. + self._manager.set( + 'forbidden', 'test.example.com', + 'http://localhost:8180/welcome_3.txt') + self.assertRaises(URLError, self._loader.get, 'forbidden', self._mlist) + + +# Response texts. +WELCOME_1 = """\ +Welcome to the {fqdn_listname} mailing list! +""" + +WELCOME_2 = """\ +Sure, I guess you're welcome. +""" + +WELCOME_3 = """\ +Well? Come. +""" + +WELCOME_4 = """\ +This is a domain welcome. +""" + +TEXTS = { + '/welcome_1.txt': WELCOME_1, + '/welcome_2.txt': WELCOME_2, + '/welcome_3.txt': WELCOME_3, + '/example.com/welcome_4.txt': WELCOME_4, + } diff --git a/src/mailman/mta/deliver.py b/src/mailman/mta/deliver.py index 6ca743df4..caca18aef 100644 --- a/src/mailman/mta/deliver.py +++ b/src/mailman/mta/deliver.py @@ -102,11 +102,11 @@ def deliver(mlist, msg, msgdata): ) template = config.logging.smtp.every if template.lower() != 'no': - log.info('%s', expand(template, substitutions)) + log.info('%s', expand(template, mlist, substitutions)) if refused: template = config.logging.smtp.refused if template.lower() != 'no': - log.info('%s', expand(template, substitutions)) + log.info('%s', expand(template, mlist, substitutions)) else: # Log the successful post, but if it was not destined to the mailing # list (e.g. to the owner or admin), print the actual recipients @@ -117,7 +117,7 @@ def deliver(mlist, msg, msgdata): substitutions['recips'] = COMMA.join(recips) template = config.logging.smtp.success if template.lower() != 'no': - log.info('%s', expand(template, substitutions)) + log.info('%s', expand(template, mlist, substitutions)) # Process any failed deliveries. temporary_failures = [] permanent_failures = [] @@ -145,7 +145,7 @@ def deliver(mlist, msg, msgdata): smtpcode = code, # noqa smtpmsg = smtp_message, # noqa ) - log.info('%s', expand(template, substitutions)) + log.info('%s', expand(template, mlist, substitutions)) # Return the results if temporary_failures or permanent_failures: raise SomeRecipientsFailed(temporary_failures, permanent_failures) diff --git a/src/mailman/mta/docs/decorating.rst b/src/mailman/mta/docs/decorating.rst index 94331163b..551256426 100644 --- a/src/mailman/mta/docs/decorating.rst +++ b/src/mailman/mta/docs/decorating.rst @@ -30,7 +30,7 @@ We start by writing the site-global header and footer template. >>> os.makedirs(site_dir) >>> config.push('templates', """ ... [paths.testing] - ... template_dir: {0} + ... template_dir: {} ... """.format(template_dir)) >>> myheader_path = os.path.join(site_dir, 'myheader.txt') @@ -44,16 +44,19 @@ We start by writing the site-global header and footer template. ... print("""\ ... User name: $user_name ... Language: $user_language - ... Options: $user_optionsurl ... """, file=fp) Then create a mailing list which will use this header and footer. Because these are site-global templates, we can use a shorted URL. >>> mlist = create_list('test@example.com') - >>> mlist.header_uri = 'mailman:///myheader.txt' - >>> mlist.footer_uri = 'mailman:///myfooter.txt' - + >>> from mailman.interfaces.template import ITemplateManager + >>> from zope.component import getUtility + >>> manager = getUtility(ITemplateManager) + >>> manager.set('list:member:regular:header', mlist.list_id, + ... 'mailman:///myheader.txt') + >>> manager.set('list:member:regular:footer', mlist.list_id, + ... 'mailman:///myfooter.txt') >>> transaction.commit() >>> msg = message_from_string("""\ @@ -65,11 +68,11 @@ these are site-global templates, we can use a shorted URL. ... This is a test. ... """) - >>> recipients = set([ + >>> recipients = { ... 'aperson@example.com', ... 'bperson@example.com', ... 'cperson@example.com', - ... ]) + ... } >>> msgdata = dict( ... recipients=recipients, @@ -80,7 +83,6 @@ More information is included when the recipient is a member of the mailing list. :: - >>> from zope.component import getUtility >>> from mailman.interfaces.member import MemberRole >>> from mailman.interfaces.usermanager import IUserManager >>> user_manager = getUtility(IUserManager) @@ -126,7 +128,6 @@ The decorations happen when the message is delivered. This is a test. User name: Anne Person Language: English (USA) - Options: http://example.com/aperson@example.com ---------- From: aperson@example.org To: test@example.com @@ -144,7 +145,6 @@ The decorations happen when the message is delivered. This is a test. User name: Bart Person Language: English (USA) - Options: http://example.com/bperson@example.com ---------- From: aperson@example.org To: test@example.com @@ -162,7 +162,6 @@ The decorations happen when the message is delivered. This is a test. User name: Cris Person Language: English (USA) - Options: http://example.com/cperson@example.com ---------- diff --git a/src/mailman/mta/tests/test_delivery.py b/src/mailman/mta/tests/test_delivery.py index fda74faad..f5729597d 100644 --- a/src/mailman/mta/tests/test_delivery.py +++ b/src/mailman/mta/tests/test_delivery.py @@ -25,10 +25,12 @@ import unittest from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.mailinglist import Personalization +from mailman.interfaces.template import ITemplateManager from mailman.mta.deliver import Deliver from mailman.testing.helpers import ( specialized_message_from_string as mfs, subscribe) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility # Global test capture. @@ -50,6 +52,7 @@ class TestIndividualDelivery(unittest.TestCase): """Test personalized delivery details.""" layer = ConfigLayer + maxDiff = None def setUp(self): self._mlist = create_list('test@example.com') @@ -70,22 +73,21 @@ Subject: test path = os.path.join(self._template_dir, 'site', 'en', 'member-footer.txt') os.makedirs(os.path.dirname(path)) - with open(path, 'w') as fp: + with open(path, 'w', encoding='utf-8') as fp: print("""\ address : $user_address delivered: $user_delivered_to language : $user_language name : $user_name -options : $user_optionsurl """, file=fp) config.push('templates', """ [paths.testing] - template_dir: {0} + template_dir: {} """.format(self._template_dir)) self.addCleanup(config.pop, 'templates') - self._mlist.footer_uri = 'mailman:///member-footer.txt' - # Let assertMultiLineEqual work without bounds. - self.maxDiff = None + getUtility(ITemplateManager).set( + 'list:member:regular:footer', self._mlist.list_id, + 'mailman:///member-footer.txt') def tearDown(self): # Free global references. @@ -123,6 +125,5 @@ address : anne@example.org delivered: anne@example.org language : English (USA) name : Anne Person -options : http://example.com/anne@example.org """) diff --git a/src/mailman/mta/verp.py b/src/mailman/mta/verp.py index 87c6609a0..8ede6cb67 100644 --- a/src/mailman/mta/verp.py +++ b/src/mailman/mta/verp.py @@ -64,7 +64,7 @@ class VERPMixin: recipient) return sender return '{0}@{1}'.format( - expand(config.mta.verp_format, dict( + expand(config.mta.verp_format, mlist, dict( bounces=sender_mailbox, local=recipient_mailbox, domain=DOT.join(recipient_domain))), diff --git a/src/mailman/rest/docs/__init__.py b/src/mailman/rest/docs/__init__.py index 113e299c3..5a039ac3d 100644 --- a/src/mailman/rest/docs/__init__.py +++ b/src/mailman/rest/docs/__init__.py @@ -17,8 +17,103 @@ """Doctest layer setup.""" +import threading + +from http.server import BaseHTTPRequestHandler, HTTPServer from mailman import public +from mailman.testing.helpers import wait_for_webservice from mailman.testing.layers import RESTLayer -public(layer=RESTLayer) +# New in Python 3.5. +try: + from http import HTTPStatus +except ImportError: # pragma: no cover + class HTTPStatus: + FORBIDDEN = 403 + NOT_FOUND = 404 + OK = 200 + + +# We need a web server to vend non-mailman: urls. +class TestableHandler(BaseHTTPRequestHandler): + # Be quiet. + def log_request(*args, **kws): + pass + + log_error = log_request + + def do_GET(self): # pragma: no cover + if self.path == '/welcome_2.txt': + if self.headers['Authorization'] != 'Basic YW5uZTppcyBzcGVjaWFs': + self.send_error(HTTPStatus.FORBIDDEN) + return + response = TEXTS.get(self.path) + if response is None: + self.send_error(HTTPStatus.NOT_FOUND) # pragma: no cover + return + self.send_response(HTTPStatus.OK) + self.send_header('Content-Type', 'UTF-8') + self.end_headers() + self.wfile.write(response.encode('utf-8')) + + +class HTTPLayer(RESTLayer): + httpd = None + + @classmethod + def setUp(cls): + assert cls.httpd is None, 'Layer already set up' + cls.httpd = HTTPServer(('localhost', 8180), TestableHandler) + cls._thread = threading.Thread(target=cls.httpd.serve_forever) + cls._thread.daemon = True + cls._thread.start() + wait_for_webservice('localhost', 8180) + + @classmethod + def tearDown(cls): + assert cls.httpd is not None, 'Layer not set up' + cls.httpd.shutdown() + cls.httpd.server_close() + cls._thread.join() + + +public(layer=HTTPLayer) + + +# Response texts. +WELCOME_1 = """\ +Welcome to the "$list_name" mailing list! + +To post to this list, send your email to: + + $fqdn_listname + +There is a Code of Conduct for this mailing list which you can view at +http://www.example.com/code-of-conduct.html +""" + +WELCOME_2 = """\ +I'm glad you made it! +""" + +WELCOME_3 = """\ +Je suis heureux que vous pouvez nous rejoindre! +""" + +WELCOME_4 = """\ +Welcome to the $list_name list in the $domain domain. +""" + +WELCOME_5 = """\ +Yay! You joined the $fqdn_listname mailing list. +""" + + +TEXTS = { + '/welcome_1.txt': WELCOME_1, + '/welcome_2.txt': WELCOME_2, + '/ant.example.com/fr/welcome_3.txt': WELCOME_3, + '/welcome_4.txt': WELCOME_4, + '/welcome_5.txt': WELCOME_5, + } diff --git a/src/mailman/rest/docs/domains.rst b/src/mailman/rest/docs/domains.rst index 89c03d170..82b7dc7cd 100644 --- a/src/mailman/rest/docs/domains.rst +++ b/src/mailman/rest/docs/domains.rst @@ -26,19 +26,16 @@ initially none. Once a domain is added, it is accessible through the API. :: - >>> domain_manager.add( - ... 'example.com', 'An example domain', 'http://lists.example.com') - + >>> domain_manager.add('example.com', 'An example domain') + >>> transaction.commit() >>> dump_json('http://localhost:9001/3.0/domains') entry 0: - base_url: http://lists.example.com description: An example domain http_etag: "..." mail_host: example.com self_link: http://localhost:9001/3.0/domains/example.com - url_host: lists.example.com http_etag: "..." start: 0 total_size: 1 @@ -46,39 +43,30 @@ Once a domain is added, it is accessible through the API. At the top level, all domains are returned as separate entries. :: - >>> domain_manager.add( - ... 'example.org', - ... base_url='http://mail.example.org') - + >>> domain_manager.add('example.org',) + >>> domain_manager.add( ... 'lists.example.net', - ... 'Porkmasters', - ... 'http://example.net') - + ... 'Porkmasters') + >>> transaction.commit() >>> dump_json('http://localhost:9001/3.0/domains') entry 0: - base_url: http://lists.example.com description: An example domain http_etag: "..." mail_host: example.com self_link: http://localhost:9001/3.0/domains/example.com - url_host: lists.example.com entry 1: - base_url: http://mail.example.org description: None http_etag: "..." mail_host: example.org self_link: http://localhost:9001/3.0/domains/example.org - url_host: mail.example.org entry 2: - base_url: http://example.net description: Porkmasters http_etag: "..." mail_host: lists.example.net self_link: http://localhost:9001/3.0/domains/lists.example.net - url_host: example.net http_etag: "..." start: 0 total_size: 3 @@ -91,12 +79,10 @@ The information for a single domain is available by following one of the ``self_links`` from the above collection. >>> dump_json('http://localhost:9001/3.0/domains/lists.example.net') - base_url: http://example.net description: Porkmasters http_etag: "..." mail_host: lists.example.net self_link: http://localhost:9001/3.0/domains/lists.example.net - url_host: example.net You can also list all the mailing lists for a given domain. At first, the example.com domain does not contain any mailing lists. @@ -154,30 +140,26 @@ New domains can be created by posting to the ``domains`` url. Now the web service knows about our new domain. >>> dump_json('http://localhost:9001/3.0/domains/lists.example.com') - base_url: http://lists.example.com description: None http_etag: "..." mail_host: lists.example.com self_link: http://localhost:9001/3.0/domains/lists.example.com - url_host: lists.example.com And the new domain is in our database. :: >>> domain_manager['lists.example.com'] - + # Unlock the database. >>> transaction.abort() -You can also create a new domain with a description, a base url, and a contact -address. +You can also create a new domain with a description and a contact address. :: >>> dump_json('http://localhost:9001/3.0/domains', { ... 'mail_host': 'my.example.com', ... 'description': 'My new domain', - ... 'base_url': 'http://allmy.example.com' ... }) content-length: 0 content-type: application/json; charset=UTF-8 @@ -186,15 +168,13 @@ address. ... >>> dump_json('http://localhost:9001/3.0/domains/my.example.com') - base_url: http://allmy.example.com description: My new domain http_etag: "..." mail_host: my.example.com self_link: http://localhost:9001/3.0/domains/my.example.com - url_host: allmy.example.com >>> domain_manager['my.example.com'] - + # Unlock the database. >>> transaction.abort() diff --git a/src/mailman/rest/docs/listconf.rst b/src/mailman/rest/docs/listconf.rst index 6804644ac..922da5ea4 100644 --- a/src/mailman/rest/docs/listconf.rst +++ b/src/mailman/rest/docs/listconf.rst @@ -36,6 +36,8 @@ All readable attributes for a list are available on a sub-resource. default_member_action: defer default_nonmember_action: hold description: + digest_footer_uri: + digest_header_uri: digest_last_sent_at: None digest_send_periodic: True digest_size_threshold: 30.0 @@ -44,8 +46,10 @@ All readable attributes for a list are available on a sub-resource. display_name: Ant filter_content: False first_strip_reply_to: False + footer_uri: fqdn_listname: ant@example.com goodbye_message_uri: + header_uri: http_etag: "..." include_rfc2369_headers: True join_address: ant-join@example.com @@ -63,13 +67,11 @@ All readable attributes for a list are available on a sub-resource. reply_goes_to_list: no_munging reply_to_address: request_address: ant-request@example.com - scheme: http send_welcome_message: True subject_prefix: [Ant] subscription_policy: confirm volume: 1 - web_host: lists.example.com - welcome_message_uri: mailman:///welcome.txt + welcome_message_uri: Changing the full configuration @@ -109,7 +111,6 @@ When using ``PUT``, all writable attributes must be included. ... posting_pipeline='virgin', ... filter_content=True, ... first_strip_reply_to=True, - ... goodbye_message_uri='mailman:///goodbye.txt', ... convert_html_to_plaintext=True, ... collapse_alternatives=False, ... reply_goes_to_list='point_to_list', @@ -117,7 +118,6 @@ When using ``PUT``, all writable attributes must be included. ... send_welcome_message=False, ... subject_prefix='[ant]', ... subscription_policy='moderate', - ... welcome_message_uri='mailman:///welcome.txt', ... default_member_action='hold', ... default_nonmember_action='discard', ... moderator_password='password', @@ -162,8 +162,8 @@ These values are changed permanently. display_name: Fnords filter_content: True first_strip_reply_to: True + footer_uri: fqdn_listname: ant@example.com - goodbye_message_uri: mailman:///goodbye.txt ... include_rfc2369_headers: False ... @@ -177,7 +177,6 @@ These values are changed permanently. subject_prefix: [ant] subscription_policy: moderate ... - welcome_message_uri: mailman:///welcome.txt Changing a partial configuration diff --git a/src/mailman/rest/docs/systemconf.rst b/src/mailman/rest/docs/systemconf.rst index fa8b7384b..385588077 100644 --- a/src/mailman/rest/docs/systemconf.rst +++ b/src/mailman/rest/docs/systemconf.rst @@ -12,6 +12,7 @@ get a list of all defined sections. You can also get all the values for a particular section. >>> dump_json('http://localhost:9001/3.0/system/configuration/mailman') + cache_life: 7d default_language: en email_commands_max_lines: 10 filtered_messages_are_preservable: no diff --git a/src/mailman/rest/docs/templates.rst b/src/mailman/rest/docs/templates.rst new file mode 100644 index 000000000..f5d6773dc --- /dev/null +++ b/src/mailman/rest/docs/templates.rst @@ -0,0 +1,539 @@ +=========== + Templates +=========== + +In Mailman 3.1 a new template system was introduced to allow for maximum +flexibility in the format and content of messages sent by and through Mailman. +For example, when a new member joins a list, a welcome message is sent to that +member. The welcome message is created from a template found by a URL +associated with a template name and a context. + +So if for example, you want to include links to pages on you website, you can +create a custom template, make it available via download from a URL, and then +associate that URL with a mailing list's welcome message. Some standard +placeholders can be defined in the template, and these will be filled in by +Mailman when the welcome message is sent. + +The URL itself can have placeholders, and this allows for additional +flexibility when looking up the content. + + +Examples +======== + +Let's say you have a mailing list:: + + >>> ant = create_list('ant@example.com') + +The standard welcome message doesn't have any links to it because by default +Mailman doesn't know about any web user interface front-end. When Anne is +subscribed to the mailing list, she sees this plain welcome message. + + >>> anne = subscribe(ant, 'Anne') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Ant" mailing list + From: ant-request@example.com + To: Anne Person + ... + + Welcome to the "Ant" mailing list! + + To post to this list, send your email to: + + ant@example.com + + You can make such adjustments via email by sending a message to: + + ant-request@example.com + + with the word 'help' in the subject or body (don't include the + quotes), and you will get back a message with instructions. You will + need your password to change your options, but for security purposes, + this email is not included here. There is also a button on your + options page that will send your current password to you. + +Let's say though that you wanted to provide a link to a Code of Conduct in the +welcome message. You publish both the code of conduct and the welcome message +pointing to the code on your website. Now you can tell the mailing list to +use this welcome message instead of the default one. + + >>> call_http('http://localhost:9001/3.1/lists/ant.example.com/uris', { + ... 'list:user:notice:welcome': 'http://localhost:8180/welcome_1.txt', + ... }, method='PATCH') + content-length: 0 + date: ... + server: ... + status: 204 + +The name of the template corresponding to the welcome message is +`list:user:notice:welcome` and the location of your new welcome message text +is at `http://localhost:8080/welcome_1.txt`. + +Now when a new member subscribes to the mailing list, they'll see the new +welcome message. + + >>> bill = subscribe(ant, 'Bill') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Ant" mailing list + From: ant-request@example.com + To: Bill Person + ... + + Welcome to the "Ant" mailing list! + + To post to this list, send your email to: + + ant@example.com + + There is a Code of Conduct for this mailing list which you can view at + http://www.example.com/code-of-conduct.html + +It's even possible to require a username and password (Basic Auth) for +retrieving the welcome message. + + >>> call_http('http://localhost:9001/3.1/lists/ant.example.com/uris', { + ... 'list:user:notice:welcome': 'http://localhost:8180/welcome_2.txt', + ... 'username': 'anne', + ... 'password': 'is special', + ... }, method='PATCH') + content-length: 0 + date: ... + server: ... + status: 204 + +The username and password will be used to retrieve the welcome text. + + >>> cris = subscribe(ant, 'Cris') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Ant" mailing list + From: ant-request@example.com + To: Cris Person + ... + + I'm glad you made it! + +The text is cached so subsequent uses don't necessarily need to hit the +internet. + + >>> dave = subscribe(ant, 'Dave') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Ant" mailing list + From: ant-request@example.com + To: Dave Person + ... + + I'm glad you made it! + + +Template format +=============== + +Mailman expects the templates to be return as content type +`text/plain; charset="UTF-8"`. + +Template URLs can be any of the following schemes: + +* `http://` - standard scheme supported by the requests_ library; +* `https://` - standard scheme also supported by requests_; +* `file:///` - any path on the local file system; UTF-8 contents by default; +* `mailman:///` - a path defined within the Mailman source code tree. It is + not recommended that you use these; they are primarily provided for + `Mailman's internal use`_. + +Generally, if a template is not defined or not found, the empty string is +used. IOW, a missing template does not cause an error, it simply causes the +named template to be blank. + + +URL placeholders +================ + +The URLs themselves can contain placeholders, and this can be used to provide +even more flexibility in the way the template texts are retrieved. Two common +placeholders include the List-ID and the mailing list's preferred language +code. + + >>> ant.preferred_language = 'fr' + >>> call_http('http://localhost:9001/3.1/lists/ant.example.com/uris', { + ... 'list:user:notice:welcome': + ... 'http://localhost:8180/$list_id/$language/welcome_3.txt', + ... }, method='PATCH') + content-length: 0 + date: ... + server: ... + status: 204 + +The next person to subscribe will get a French welcome message. + + >>> dave = subscribe(ant, 'Elle') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + Subject: =?iso-8859-1?q?Welcome_to_the_=22Ant=22_mailing_list?= + From: ant-request@example.com + To: Elle Person + ... + + Je suis heureux que vous pouvez nous rejoindre! + +Standard URL substitutions include: + +* `$list_id` - The mailing list's List-ID (`ant.example.com`) +* `$listname` - The mailing list's fully qualified list name + (`ant@example.com`) +* `$domain_name` - The mailing list's domain name (`example.com`) +* `$language` - The language code for the mailing list's preferred language + (`fr`) + + +Template contexts +================= + +When Mailman is looking for a template, it always searches for it in up to +three *contexts*, and you can set the template for any of these three +contexts: a mailing list, a domain, the site. + +Most templates are searched first by the mailing list, then by domain, then by +site. One notable exception is the ``domain:admin:notice:new-list`` template, +which is sent when a new mailing list is created. Because (modulo any style +default settings) there won't be a template for the newly created mailing +list, this template is always searched for first in the domain, and then in +the site. + +In fact, this illustrates a common naming scheme for templates. The +colon-separated sections usually follow the form +``:::`` where ``context`` would be "domain" or +"list, ```` would be "admin", "user", or "member", and ```` +can be "action" or "notice". This isn't a strict naming scheme, but it does +give you some indication as to the use of the template. All template names +used internally by Mailman are given below. + +You've already seen how the mailing list context works above. Let's look at +the domain and site contexts next. + + +Domain context +-------------- + +Let's say you want all mailing lists in a given domain to share exactly the +same welcome message template. Remember that Mailman will insert +substitutions into the templates themselves to customize them for each mailing +list, so in general a single template can be shared by all mailing lists in +the domain. + +The first thing to do is to set the URI for the welcome message in the domain +to be shared. + + >>> call_http('http://localhost:9001/3.1/domains/example.com/uris', { + ... 'list:user:notice:welcome': + ... 'http://localhost:8180/welcome_4.txt', + ... }, method='PATCH') + content-length: 0 + date: ... + server: ... + status: 204 + +And let's create a new mailing list in this domain. + + >>> bee = create_list('bee@example.com') + +Now when Anne subscribes to the Bee mailing list, she will get this +domain-wide welcome message. + + >>> anne = subscribe(bee, 'Anne') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Bee" mailing list + From: bee-request@example.com + To: Anne Person + ... + Welcome to the Bee list in the example.com domain. + +So far so good. What happens if Fred subscribes to the Ant mailing list? + + >>> fred = subscribe(ant, 'Fred') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + Subject: =?iso-8859-1?q?Welcome_to_the_=22Ant=22_mailing_list?= + From: ant-request@example.com + To: Fred Person + ... + + Je suis heureux que vous pouvez nous rejoindre! + +Okay, that's strange! Why did Fred get the French welcome message? It's +because the mailing list context overrides the domain context! Similarly, a +domain context overrides a site context. This allows you to provide generic +templates to be used as a default, with specific overrides where necessary. + +Let's delete the Ant list's override. + + >>> ant.preferred_language = 'en' + >>> call_http('http://localhost:9001/3.1/lists/ant.example.com/uris' + ... '/list:user:notice:welcome', + ... method='DELETE') + content-length: 0 + date: ... + server: ... + status: 204 + +Now when Gwen subscribes to the Ant list, she gets the domain's welcome +message. + + >>> gwen = subscribe(ant, 'Gwen') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Ant" mailing list + From: ant-request@example.com + To: Gwen Person + ... + + Welcome to the Ant list in the example.com domain. + + +Site context +------------ + +Let's say we want the same welcome template for every mailing list on our +Mailman installation. For this we use the site context. + +First, let's delete the domain context we set previously. Note that +previously we used a `DELETE` method on the list's welcome template resource, +but we could have also done this by PATCHing an empty string for the URI, +which Mailman's REST API interprets as a deletion too. Let's use this +approach to delete the domain welcome message. + + >>> call_http('http://localhost:9001/3.1/domains/example.com/uris', { + ... 'list:user:notice:welcome': '', + ... }, method='PATCH') + content-length: 0 + date: ... + server: ... + status: 204 + +Now let's set a new welcome template URI for the site. + + >>> call_http('http://localhost:9001/3.1/uris', { + ... 'list:user:notice:welcome': + ... 'http://localhost:8180/welcome_5.txt', + ... }, method='PATCH') + content-length: 0 + date: ... + server: ... + status: 204 + +Now Herb subscribes to both the Ant... + + >>> herb = subscribe(ant, 'Herb') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Ant" mailing list + From: ant-request@example.com + To: Herb Person + ... + + Yay! You joined the ant@example.com mailing list. + +...and Bee mailing lists. + + >>> herb = subscribe(bee, 'Herb') + >>> items = get_queue_messages('virgin') + >>> print(items[0].msg) + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Welcome to the "Bee" mailing list + From: bee-request@example.com + To: Herb Person + ... + + Yay! You joined the bee@example.com mailing list. + + +Templated texts +=============== + +All the texts that Mailman uses to create or decorate messages can be +associated with a URL. Mailman looks up templates by name and downloads it +via that URL. The retrieved text supports placeholders which are filled in by +Mailman. There are a common set of placeholders most templates support: + +* ``listname`` - fully qualified list name (e.g. ``ant@example.com``) +* ``list_id`` - the ``List-ID`` header (e.g. ``ant.example.com``) +* ``display_name`` - the display name of the mailing list (e.g. ``Ant``) +* ``short_listname`` - the local part of the list name (e.g. ``ant``) +* ``domain`` - the domain name part of the list name (e.g. ``example.com``) +* ``description`` - the mailing list's short description text +* ``info`` - the mailing list's longer descriptive text +* ``request_email`` - the email address for the ``-request`` alias +* ``owner_email`` - the email address for the ``-owner`` alias +* ``site_email`` - the email address to reach the owners of the site +* ``language`` - the two letter language code for the list's preferred + language (e.g. ``en``, ``it``, ``fr``) + +Other template substitutions are described below the template name listed +below. Here are all the supported template names: + +* ``domain:admin:notice:new-list`` + Sent to the administrators of any newly created mailing list. + +* ``list:admin:action:post`` + Sent to the list administrators when moderator approval for a posting is + required. + + * ``subject`` - the original ``Subject`` of the message + * ``sender_email`` - the poster's email address + * ``reasons`` - some reasons why the post is being held for approval + +* ``list:admin:action:subscribe`` + Sent to the list administrators when moderator approval for a subscription + request is required. + + * ``member`` - display name and email address of the subscriber + +* ``list:admin:action:unsubscribe`` + Sent to the list administrators when moderator approval for an + unsubscription request is required. + + * ``member`` - display name and email address of the subscriber + +* ``list:admin:notice:subscribe`` + Sent to the list administrators to notify them when a new member has + been subscribed. + + * ``member`` - display name and email address of the subscriber + +* ``list:admin:notice:unrecognized`` + Sent to the list administrators when a bounce message in an unrecognized + format has been received. + +* ``list:admin:notice:unsubscribe`` + Sent to the list administrators to notify them when a member has been + unsubscribed. + + * ``member`` - display name and email address of the subscriber + +* ``list:member:digest:footer`` + The footer for a digest message. + +* ``list:member:digest:header`` + The header for a digest message. + +* ``list:member:digest:masthead`` + The digest "masthead"; i.e. a common introduction for all digest + messages. + +* ``list:member:regular:footer`` + The footer for a regular (non-digest) message. + + When personalized deliveries are enabled, these substitution variables are + also defined: + + * ``member`` - display name and email address of the subscriber + * ``user_email`` - the email address of the recipient + * ``user_delivered_to`` - the case-preserved email address of the recipient + * ``user_language`` - the description of the user's preferred language + (e.g. "French", "English", "Italian") + * ``user_name`` - the recipient's display name if available + +* ``list:member:regular:header`` + The header for a regular (non-digest) message. + + When personalized deliveries are enabled, these substitution variables are + also defined: + + * ``member`` - display name and email address of the subscriber + * ``user_email`` - the email address of the recipient + * ``user_delivered_to`` - the case-preserved email address of the recipient + * ``user_language`` - the description of the user's preferred language + (e.g. "French", "English", "Italian") + * ``user_name`` - the recipient's display name if available + +* ``list:user:action:confirm`` + The message sent to subscribers when a subscription confirmation is + required. + + * ``token`` - the unique confirmation token + * ``subject`` - the ``Subject`` heading for the confirmation email, which + includes the confirmation token + * ``confirm_email`` - the email address to send the confirmation response + to; this corresponds to the ``Reply-To`` header + * ``user_email`` - the email address being confirmed + +* ``list:user:notice:goodbye`` + The notice sent to a member when they unsubscribe from a mailing list. + +* ``list:user:notice:hold`` + The notice sent to a poster when their message is being held or moderator + approval. + + * ``subject`` - the original ``Subject`` of the message + * ``sender_email`` - the poster's email address + * ``reasons`` - some reasons why the post is being held for approval + +* ``list:user:notice:no-more-today`` + Sent to a user when the maximum number of autoresponses has been reached + for that day. + + * ``sender_email`` - the email address of the poster + * ``count`` - the number of autoresponse messages sent to the user today + +* ``list:user:notice:post`` + Notice sent to a poster when their message has been received by the + mailing list. + + * ``subject`` - the ``Subject`` field of the received message + +* ``list:user:notice:probe`` + A bounce probe sent to a member when their subscription has been disabled + due to bounces. + + * ``sender_email`` - the email address of the bouncing member + +* ``list:user:notice:refuse`` + Notice sent to a poster when their message has been rejected by the list's + moderator. + + * ``request`` - the type of request being rejected + * ``reason`` - the reason for the rejection, as provided by the list's + moderators + +* ``list:user:notice:welcome`` + The notice sent to a member when they are subscribed to the mailing list. + + * ``user_name`` - the display name of the new member + * ``user_email`` - the email address of the new member + + +.. _requests: http://docs.python-requests.org/en/master/ +.. _`Mailman's internal use`: https://gitlab.com/mailman/mailman/blob/master/src/mailman/utilities/i18n.py#L45 diff --git a/src/mailman/rest/domains.py b/src/mailman/rest/domains.py index 62e05b82f..7f54e8905 100644 --- a/src/mailman/rest/domains.py +++ b/src/mailman/rest/domains.py @@ -24,6 +24,7 @@ from mailman.rest.helpers import ( BadRequest, CollectionMixin, NotFound, bad_request, child, created, etag, no_content, not_found, okay) from mailman.rest.lists import ListsForDomain +from mailman.rest.uris import ADomainURI, AllDomainURIs from mailman.rest.users import OwnersForDomain from mailman.rest.validator import Validator, list_of_strings_validator from zope.component import getUtility @@ -35,11 +36,9 @@ class _DomainBase(CollectionMixin): def _resource_as_dict(self, domain): """See `CollectionMixin`.""" return dict( - base_url=domain.base_url, description=domain.description, mail_host=domain.mail_host, self_link=self.api.path_to('domains/{}'.format(domain.mail_host)), - url_host=domain.url_host, ) def _get_collection(self, request): @@ -94,6 +93,24 @@ class ADomain(_DomainBase): else: return NotFound(), [] + @child() + def uris(self, context, segments): + """Return the template URIs of the domain. + + These are only available after API 3.0. + """ + domain = getUtility(IDomainManager).get(self._domain) + if domain is None or self.api.version_info < (3, 1): + return NotFound(), [] + if len(segments) == 0: + return AllDomainURIs(domain) + if len(segments) > 1: + return BadRequest(), [] + template = segments[0] + if template not in AllDomainURIs.URIs: + return NotFound(), [] + return ADomainURI(domain, template), [] + @public class AllDomains(_DomainBase): @@ -105,10 +122,8 @@ class AllDomains(_DomainBase): try: validator = Validator(mail_host=str, description=str, - base_url=str, owner=list_of_strings_validator, - _optional=( - 'description', 'base_url', 'owner')) + _optional=('description', 'owner')) values = validator(request) # For consistency, owners are passed in as multiple `owner` keys, # but .add() requires an `owners` keyword. Match impedence. diff --git a/src/mailman/rest/header_matches.py b/src/mailman/rest/header_matches.py index a9ccd2e35..25c519ef3 100644 --- a/src/mailman/rest/header_matches.py +++ b/src/mailman/rest/header_matches.py @@ -82,7 +82,7 @@ class HeaderMatch(_HeaderMatchBase): else: no_content(response) - def patch_put(self, request, response, is_optional): + def _patch_put(self, request, response, is_optional): """Update the header match.""" try: header_match = self.header_matches[self._position] @@ -118,11 +118,11 @@ class HeaderMatch(_HeaderMatchBase): def on_put(self, request, response): """Full update of the header match.""" - self.patch_put(request, response, is_optional=False) + self._patch_put(request, response, is_optional=False) def on_patch(self, request, response): """Partial update of the header match.""" - self.patch_put(request, response, is_optional=True) + self._patch_put(request, response, is_optional=True) @public diff --git a/src/mailman/rest/listconf.py b/src/mailman/rest/listconf.py index 3294ba8b6..0982e7970 100644 --- a/src/mailman/rest/listconf.py +++ b/src/mailman/rest/listconf.py @@ -25,12 +25,14 @@ from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( - IAcceptableAliasSet, ReplyToMunging, SubscriptionPolicy) + IAcceptableAliasSet, IMailingList, ReplyToMunging, SubscriptionPolicy) +from mailman.interfaces.template import ITemplateManager from mailman.rest.helpers import ( GetterSetter, bad_request, etag, no_content, not_found, okay) from mailman.rest.validator import ( PatchValidator, ReadOnlyPATCHRequestError, UnknownPATCHRequestError, Validator, enum_validator, list_of_strings_validator) +from zope.component import getUtility class AcceptableAliases(GetterSetter): @@ -58,6 +60,31 @@ class AcceptableAliases(GetterSetter): alias_set.add(alias) +TEMPLATE_ATTRIBUTES = dict( + digest_footer_uri='list:digest:footer', + digest_header_uri='list:digest:header', + footer_uri='list:regular:footer', + goodbye_message_uri='user:ack:goodbye', + header_uri='list:regular:header', + welcome_message_uri='user:ack:welcome', + ) + + +class URIAttributeMapper(GetterSetter): + """Map old IMailingList uri attributes to the new template manager.""" + + def get(self, obj, attribute): + assert IMailingList.providedBy(obj), obj + template_name = TEMPLATE_ATTRIBUTES[attribute] + template = getUtility(ITemplateManager).raw(template_name, obj.list_id) + return '' if template is None else template.uri + + def put(self, obj, attribute, value): + assert IMailingList.providedBy(obj), obj + template_name = TEMPLATE_ATTRIBUTES[attribute] + getUtility(ITemplateManager).set(template_name, obj.list_id, value) + + # Additional validators for converting from web request strings to internal # data types. See below for details. @@ -117,7 +144,6 @@ ATTRIBUTES = dict( digests_enabled=GetterSetter(as_boolean), filter_content=GetterSetter(as_boolean), first_strip_reply_to=GetterSetter(as_boolean), - goodbye_message_uri=GetterSetter(str), fqdn_listname=GetterSetter(None), mail_host=GetterSetter(None), allow_list_posts=GetterSetter(as_boolean), @@ -137,13 +163,10 @@ ATTRIBUTES = dict( reply_goes_to_list=GetterSetter(enum_validator(ReplyToMunging)), reply_to_address=GetterSetter(str), request_address=GetterSetter(None), - scheme=GetterSetter(None), send_welcome_message=GetterSetter(as_boolean), subject_prefix=GetterSetter(str), subscription_policy=GetterSetter(enum_validator(SubscriptionPolicy)), volume=GetterSetter(None), - web_host=GetterSetter(None), - welcome_message_uri=GetterSetter(str), ) @@ -153,6 +176,20 @@ for attribute, gettersetter in list(VALIDATORS.items()): del VALIDATORS[attribute] +def api_attributes(api): + # The list of readable attributes is different depending on the API being + # requested. Specifically, in API 3.0 the templates are exposed as list + # attributes, although we map them to templates. In API 3.1 and beyond, + # only the template manager API can be used for these. + attributes = ATTRIBUTES.copy() + if api.version_info == (3, 0): + attributes.update({ + attribute: URIAttributeMapper(str) + for attribute in TEMPLATE_ATTRIBUTES + }) + return attributes + + @public class ListConfiguration: """A mailing list configuration resource.""" @@ -164,31 +201,44 @@ class ListConfiguration: def on_get(self, request, response): """Get a mailing list configuration.""" resource = {} + attributes = api_attributes(self.api) if self._attribute is None: - # This is a requst for all the mailing list's configuration + # This is a request for all the mailing list's configuration # variables. Return all readable attributes. - for attribute in ATTRIBUTES: - value = ATTRIBUTES[attribute].get(self._mlist, attribute) + for attribute, getter in attributes.items(): + value = getter.get(self._mlist, attribute) resource[attribute] = value - elif self._attribute not in ATTRIBUTES: + elif self._attribute in attributes: + # This is a request for a specific attribute. + value = attributes[self._attribute].get( + self._mlist, self._attribute) + resource[self._attribute] = value + else: # This is a request for a specific, nonexistent attribute. not_found( response, 'Unknown attribute: {}'.format(self._attribute)) return - else: - # This is a request for a specific attribute. - attribute = self._attribute - value = ATTRIBUTES[attribute].get(self._mlist, attribute) - resource[attribute] = value okay(response, etag(resource)) def on_put(self, request, response): """Set a mailing list configuration.""" attribute = self._attribute + # The list of required attributes differs between API version. For + # backward compatibility, in API 3.0 all of the *_uri attributes are + # optional. In API 3.1 none of these are allowed since they are + # handled by the template manager API. + validators = VALIDATORS.copy() + attributes = api_attributes(self.api) + if self.api.version_info == (3, 0): + validators.update({ + attribute: URIAttributeMapper(str) + for attribute in TEMPLATE_ATTRIBUTES + }) + validators['_optional'] = TEMPLATE_ATTRIBUTES.keys() if attribute is None: # This is a request to update all the list's writable # configuration variables. All must be provided in the request. - validator = Validator(**VALIDATORS) + validator = Validator(**validators) try: validator.update(self._mlist, request) except ValueError as error: @@ -200,18 +250,18 @@ class ListConfiguration: # contain sufficient details, so just return it as the reason. bad_request(response, str(error)) return - elif attribute not in ATTRIBUTES: + elif attribute not in attributes: # Here we're PUTting to a specific resource, but that attribute is # bogus so the URL is considered pointing to a missing resource. not_found(response, 'Unknown attribute: {}'.format(attribute)) return - elif ATTRIBUTES[attribute].decoder is None: + elif attributes[attribute].decoder is None: bad_request( response, 'Read-only attribute: {}'.format(attribute)) return else: # We're PUTting to a specific configuration sub-resource. - validator = Validator(**{attribute: VALIDATORS[attribute]}) + validator = Validator(**{attribute: validators[attribute]}) try: validator.update(self._mlist, request) except ValueError as error: @@ -221,11 +271,12 @@ class ListConfiguration: def on_patch(self, request, response): """Patch the configuration (i.e. partial update).""" + attributes = api_attributes(self.api) if self._attribute is None: # We're PATCHing one or more of the attributes on the list's # configuration resource, so all the writable attributes are valid # candidates for updating. - converters = ATTRIBUTES + converters = attributes else: # We're PATCHing a specific list configuration attribute # sub-resource. Because the request data must be a dictionary, we @@ -237,7 +288,7 @@ class ListConfiguration: bad_request(response, 'Expected 1 attribute, got {}'.format( len(keys))) return - converter = ATTRIBUTES.get(self._attribute) + converter = attributes.get(self._attribute) if converter is None: # This is the case where the URL points to a nonexisting list # configuration attribute sub-resource. diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index 69d14e548..c3919c001 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -33,12 +33,13 @@ from mailman.interfaces.subscriptions import ISubscriptionService from mailman.rest.bans import BannedEmails from mailman.rest.header_matches import HeaderMatches from mailman.rest.helpers import ( - CollectionMixin, GetterSetter, NotFound, accepted, bad_request, child, - created, etag, no_content, not_found, okay) + BadRequest, CollectionMixin, GetterSetter, NotFound, accepted, + bad_request, child, created, etag, no_content, not_found, okay) from mailman.rest.listconf import ListConfiguration from mailman.rest.members import AMember, MemberCollection from mailman.rest.post_moderation import HeldMessages from mailman.rest.sub_moderation import SubscriptionRequests +from mailman.rest.uris import AListURI, AllListURIs from mailman.rest.validator import Validator, list_of_strings_validator from zope.component import getUtility @@ -204,6 +205,23 @@ class AList(_ListBase): return NotFound(), [] return HeaderMatches(self._mlist) + @child() + def uris(self, context, segments): + """Return the template URIs of the mailing list. + + These are only available after API 3.0. + """ + if self._mlist is None or self.api.version_info < (3, 1): + return NotFound(), [] + if len(segments) == 0: + return AllListURIs(self._mlist) + if len(segments) > 1: + return BadRequest(), [] + template = segments[0] + if template not in AllListURIs.URIs: + return NotFound(), [] + return AListURI(self._mlist, template), [] + @public class AllLists(_ListBase): diff --git a/src/mailman/rest/root.py b/src/mailman/rest/root.py index ff459bbcb..7f6be0353 100644 --- a/src/mailman/rest/root.py +++ b/src/mailman/rest/root.py @@ -34,6 +34,7 @@ from mailman.rest.members import AMember, AllMembers, FindMembers from mailman.rest.preferences import ReadOnlyPreferences from mailman.rest.queues import AQueue, AQueueFile, AllQueues from mailman.rest.templates import TemplateFinder +from mailman.rest.uris import ASiteURI, AllSiteURIs from mailman.rest.users import AUser, AllUsers, ServerOwners from zope.component import getUtility @@ -249,6 +250,9 @@ class TopLevel: Use content negotiation to context language and suffix (content-type). """ + # This resource is removed in API 3.1; use the /uris resource instead. + if self.api.version_info > (3, 0): + return NotFound(), [] if len(segments) == 3: fqdn_listname, template, language = segments elif len(segments) == 2: @@ -264,6 +268,19 @@ class TopLevel: return TemplateFinder( fqdn_listname, template, language, content_type) + @child() + def uris(self, content, segments): + if self.api.version_info < (3, 1): + return NotFound(), [] + if len(segments) == 0: + return AllSiteURIs() + if len(segments) > 1: + return BadRequest(), [] + template = segments[0] + if template not in AllSiteURIs.URIs: + return NotFound(), [] + return ASiteURI(template), [] + @child() def queues(self, context, segments): """//queues[/[/file]]""" diff --git a/src/mailman/rest/sub_moderation.py b/src/mailman/rest/sub_moderation.py index f2ec122f6..f5ef072ff 100644 --- a/src/mailman/rest/sub_moderation.py +++ b/src/mailman/rest/sub_moderation.py @@ -19,6 +19,7 @@ from mailman import public from mailman.app.moderator import send_rejection +from mailman.core.i18n import _ from mailman.interfaces.action import Action from mailman.interfaces.member import AlreadySubscribedError from mailman.interfaces.pending import IPendings @@ -27,7 +28,6 @@ from mailman.rest.helpers import ( CollectionMixin, bad_request, child, conflict, etag, no_content, not_found, okay) from mailman.rest.validator import Validator, enum_validator -from mailman.utilities.i18n import _ from zope.component import getUtility diff --git a/src/mailman/rest/tests/test_bans.py b/src/mailman/rest/tests/test_bans.py index fc096fa1b..b83bf8aef 100644 --- a/src/mailman/rest/tests/test_bans.py +++ b/src/mailman/rest/tests/test_bans.py @@ -81,3 +81,8 @@ class TestBans(unittest.TestCase): self.assertEqual(cm.exception.code, 404) self.assertEqual(cm.exception.reason, b'Email is not banned: banned@example.com') + + def test_ban_missing_mailing_list(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/lists/bee.example.com/bans') + self.assertEqual(cm.exception.code, 404) diff --git a/src/mailman/rest/tests/test_domains.py b/src/mailman/rest/tests/test_domains.py index e98f75b53..41723b07c 100644 --- a/src/mailman/rest/tests/test_domains.py +++ b/src/mailman/rest/tests/test_domains.py @@ -23,6 +23,7 @@ from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.domain import IDomainManager from mailman.interfaces.listmanager import IListManager +from mailman.interfaces.template import ITemplateManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer from urllib.error import HTTPError @@ -41,7 +42,6 @@ class TestDomains(unittest.TestCase): data = dict( mail_host='example.org', description='Example domain', - base_url='http://example.org', owner=['someone@example.com', 'secondowner@example.com'], ) content, response = call_api( @@ -163,3 +163,223 @@ class TestDomainOwners(unittest.TestCase): 'owner': 'dave@example.com', }, method='DELETE') self.assertEqual(cm.exception.code, 400) + + +class TestDomainTemplates(unittest.TestCase): + """Test /domains//uris""" + + layer = RESTLayer + + def test_no_templates_for_api_30(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/domains/example.com/uris') + self.assertEqual(cm.exception.code, 404) + + def test_no_templates_for_missing_list(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/domains/example.org/uris') + self.assertEqual(cm.exception.code, 404) + + def test_path_too_long(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/domains/example.com/uris' + '/foo/bar') + self.assertEqual(cm.exception.code, 400) + + def test_get_unknown_uri(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/domains/example.com/uris' + '/not:a:template') + self.assertEqual(cm.exception.code, 404) + + def test_get_all_uris(self): + manager = getUtility(ITemplateManager) + with transaction(): + manager.set( + 'list:user:notice:welcome', 'example.com', + 'http://example.com/welcome') + manager.set( + 'list:user:notice:goodbye', 'example.com', + 'http://example.com/goodbye', + 'a user', 'the password', + ) + resource, response = call_api( + 'http://localhost:9001/3.1/domains/example.com/uris') + self.assertEqual(response.status, 200) + self.assertEqual(resource['start'], 0) + self.assertEqual(resource['total_size'], 2) + self.assertEqual( + resource['self_link'], + 'http://localhost:9001/3.1/domains/example.com/uris') + self.assertEqual(resource['entries'], [ + {'http_etag': '"e877ff896db0f2e280660ac16b9401f7925a53b9"', + 'name': 'list:user:notice:goodbye', + 'password': 'the password', + 'self_link': ('http://localhost:9001/3.1/domains/example.com' + '/uris/list:user:notice:goodbye'), + 'uri': 'http://example.com/goodbye', + 'username': 'a user', + }, + {'http_etag': '"8dac25601c3419e98e2c05df1d962a2252b67ce6"', + 'name': 'list:user:notice:welcome', + 'self_link': ('http://localhost:9001/3.1/domains/example.com' + '/uris/list:user:notice:welcome'), + 'uri': 'http://example.com/welcome', + }]) + + def test_patch_uris(self): + resource, response = call_api( + 'http://localhost:9001/3.1/domains/example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + }, method='PATCH') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:user:notice:welcome', 'example.com') + self.assertEqual(template.uri, 'http://example.org/welcome') + self.assertIsNone(template.username) + self.assertEqual(template.password, '') + template = manager.raw('list:user:notice:goodbye', 'example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertIsNone(template.username) + self.assertEqual(template.password, '') + + def test_patch_uris_with_credentials(self): + resource, response = call_api( + 'http://localhost:9001/3.1/domains/example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'password': 'some password', + 'username': 'anne.person', + }, method='PATCH') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:user:notice:welcome', 'example.com') + self.assertEqual(template.uri, 'http://example.org/welcome') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', 'example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + + def test_patch_uris_with_partial_credentials(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/domains/example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'username': 'anne.person', + }, method='PATCH') + self.assertEqual(cm.exception.code, 400) + + def test_put_all_uris(self): + resource, response = call_api( + 'http://localhost:9001/3.1/domains/example.com/uris', { + 'domain:admin:notice:new-list': '', + 'list:admin:action:post': '', + 'list:admin:action:subscribe': '', + 'list:admin:action:unsubscribe': '', + 'list:admin:notice:subscribe': '', + 'list:admin:notice:unrecognized': '', + 'list:admin:notice:unsubscribe': '', + 'list:member:digest:footer': '', + 'list:member:digest:header': '', + 'list:member:digest:masthead': '', + 'list:member:regular:footer': 'http://example.org/footer', + 'list:member:regular:header': 'http://example.org/header', + 'list:user:action:confirm': '', + 'list:user:action:unsubscribe': '', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'list:user:notice:hold': '', + 'list:user:notice:no-more-today': '', + 'list:user:notice:post': '', + 'list:user:notice:probe': '', + 'list:user:notice:refuse': '', + 'list:user:notice:welcome': 'http://example.org/welcome', + 'password': 'some password', + 'username': 'anne.person', + }, method='PUT') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:member:digest:footer', 'example.com') + self.assertIsNone(template) + template = manager.raw('list:member:digest:header', 'example.com') + self.assertIsNone(template) + template = manager.raw('list:member:regular:footer', 'example.com') + self.assertEqual(template.uri, 'http://example.org/footer') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:member:regular:header', 'example.com') + self.assertEqual(template.uri, 'http://example.org/header') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', 'example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', 'example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + + def test_delete_all_uris(self): + manager = getUtility(ITemplateManager) + with transaction(): + manager.set( + 'list:user:notice:welcome', 'example.com', + 'http://example.com/welcome') + manager.set( + 'list:user:notice:goodbye', 'example.com', + 'http://example.com/goodbye', + 'a user', 'the password', + ) + resource, response = call_api( + 'http://localhost:9001/3.1/domains/example.com/uris', + method='DELETE') + self.assertEqual(response.status, 204) + self.assertIsNone( + manager.raw('list:user:notice:welcome', 'example.com')) + self.assertIsNone( + manager.raw('list:user:notice:goodbye', 'example.com')) + + def test_get_a_url(self): + with transaction(): + getUtility(ITemplateManager).set( + 'list:user:notice:welcome', 'example.com', + 'http://example.com/welcome') + resource, response = call_api( + 'http://localhost:9001/3.1/domains/example.com/uris' + '/list:user:notice:welcome') + self.assertEqual(response.status, 200) + self.assertEqual(resource, { + 'http_etag': '"8884a0b3d675b4cb9899a7825daac9db88b70bed"', + 'self_link': ('http://localhost:9001/3.1/domains/example.com' + '/uris/list:user:notice:welcome'), + 'uri': 'http://example.com/welcome', + }) + + def test_get_a_bad_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/domains/example.com/uris' + '/list:user:notice:notemplate') + self.assertEqual(cm.exception.code, 404) + + def test_get_unset_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/domains/example.com/uris' + '/list:user:notice:welcome') + self.assertEqual(cm.exception.code, 404) + + def test_patch_url_with_too_many_parameters(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/domains/example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'secret': 'some password', + 'person': 'anne.person', + }, method='PATCH') + self.assertEqual(cm.exception.code, 400) diff --git a/src/mailman/rest/tests/test_listconf.py b/src/mailman/rest/tests/test_listconf.py index b5d8c49f6..9bc3486be 100644 --- a/src/mailman/rest/tests/test_listconf.py +++ b/src/mailman/rest/tests/test_listconf.py @@ -24,9 +24,11 @@ from mailman.database.transaction import transaction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( IAcceptableAliasSet, SubscriptionPolicy) +from mailman.interfaces.template import ITemplateManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer from urllib.error import HTTPError +from zope.component import getUtility # The representation of the listconf resource as a dictionary. This is used # when PUTting to the list's configuration resource. @@ -373,7 +375,9 @@ class TestConfiguration(unittest.TestCase): def test_get_goodbye_message_uri(self): with transaction(): - self._mlist.goodbye_message_uri = 'mailman:///goodbye.txt' + getUtility(ITemplateManager).set( + 'user:ack:goodbye', self._mlist.list_id, + 'mailman:///goodbye.txt') resource, response = call_api( 'http://localhost:9001/3.0/lists/ant.example.com/config' '/goodbye_message_uri') @@ -387,7 +391,9 @@ class TestConfiguration(unittest.TestCase): 'PATCH') self.assertEqual(response.status, 204) self.assertEqual( - self._mlist.goodbye_message_uri, 'mailman:///salutation.txt') + getUtility(ITemplateManager).raw( + 'user:ack:goodbye', self._mlist.list_id).uri, + 'mailman:///salutation.txt') def test_patch_goodbye_message_uri(self): resource, response = call_api( @@ -397,11 +403,17 @@ class TestConfiguration(unittest.TestCase): 'PATCH') self.assertEqual(response.status, 204) self.assertEqual( - self._mlist.goodbye_message_uri, 'mailman:///salutation.txt') + getUtility(ITemplateManager).raw( + 'user:ack:goodbye', self._mlist.list_id).uri, + 'mailman:///salutation.txt') def test_put_goodbye_message_uri(self): + manager = getUtility(ITemplateManager) with transaction(): - self._mlist.goodbye_message_uri = 'mailman:///somefile.txt' + manager.set( + 'user:ack:goodbye', + self._mlist.list_id, + 'mailman:///somefile.txt') resource, response = call_api( 'http://localhost:9001/3.0/lists/ant.example.com/config' '/goodbye_message_uri', @@ -409,7 +421,8 @@ class TestConfiguration(unittest.TestCase): 'PUT') self.assertEqual(response.status, 204) self.assertEqual( - self._mlist.goodbye_message_uri, 'mailman:///salutation.txt') + manager.raw('user:ack:goodbye', self._mlist.list_id).uri, + 'mailman:///salutation.txt') def test_advertised(self): # GL issue #220 claimed advertised was read-only. diff --git a/src/mailman/rest/tests/test_lists.py b/src/mailman/rest/tests/test_lists.py index 891afa889..787009855 100644 --- a/src/mailman/rest/tests/test_lists.py +++ b/src/mailman/rest/tests/test_lists.py @@ -27,6 +27,7 @@ from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mailinglist import IAcceptableAliasSet from mailman.interfaces.member import DeliveryMode +from mailman.interfaces.template import ITemplateManager from mailman.interfaces.usermanager import IUserManager from mailman.model.mailinglist import AcceptableAlias from mailman.runners.digest import DigestRunner @@ -545,3 +546,234 @@ Subject: message 1 items = get_queue_messages('virgin') self.assertEqual(len(items), 1) self.assertEqual(items[0].msg['subject'], 'Ant Digest, Vol 8, Issue 1') + + +class TestListTemplates(unittest.TestCase): + """Test /lists//uris""" + + layer = RESTLayer + + def setUp(self): + with transaction(): + self._mlist = create_list('ant@example.com') + + def test_no_templates_for_api_30(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/lists/ant.example.com/uris') + self.assertEqual(cm.exception.code, 404) + + def test_no_templates_for_missing_list(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/lists/bee.example.com/uris') + self.assertEqual(cm.exception.code, 404) + + def test_path_too_long(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/lists/ant.example.com/uris' + '/foo/bar') + self.assertEqual(cm.exception.code, 400) + + def test_get_unknown_uri(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/lists/ant.example.com/uris' + '/not:a:template') + self.assertEqual(cm.exception.code, 404) + + def test_get_all_uris(self): + manager = getUtility(ITemplateManager) + with transaction(): + manager.set( + 'list:user:notice:welcome', 'ant.example.com', + 'http://example.com/welcome') + manager.set( + 'list:user:notice:goodbye', 'ant.example.com', + 'http://example.com/goodbye', + 'a user', 'the password', + ) + resource, response = call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris') + self.assertEqual(response.status, 200) + self.assertEqual(resource['start'], 0) + self.assertEqual(resource['total_size'], 2) + self.assertEqual( + resource['self_link'], + 'http://localhost:9001/3.1/lists/ant.example.com/uris') + self.assertEqual(resource['entries'], [ + {'http_etag': '"6612187ed6604ce54a57405fd66742557391ed4a"', + 'name': 'list:user:notice:goodbye', + 'password': 'the password', + 'self_link': ('http://localhost:9001/3.1/lists/ant.example.com' + '/uris/list:user:notice:goodbye'), + 'uri': 'http://example.com/goodbye', + 'username': 'a user', + }, + {'http_etag': '"cb1ab5eee2242143d2984edd0487532915ad3a8e"', + 'name': 'list:user:notice:welcome', + 'self_link': ('http://localhost:9001/3.1/lists/ant.example.com' + '/uris/list:user:notice:welcome'), + 'uri': 'http://example.com/welcome', + }]) + + def test_patch_uris(self): + resource, response = call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + }, method='PATCH') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:user:notice:welcome', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/welcome') + self.assertIsNone(template.username) + self.assertEqual(template.password, '') + template = manager.raw('list:user:notice:goodbye', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertIsNone(template.username) + self.assertEqual(template.password, '') + + def test_patch_uris_with_credentials(self): + resource, response = call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'password': 'some password', + 'username': 'anne.person', + }, method='PATCH') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:user:notice:welcome', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/welcome') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + + def test_patch_uris_with_partial_credentials(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'username': 'anne.person', + }, method='PATCH') + self.assertEqual(cm.exception.code, 400) + + def test_put_all_uris(self): + resource, response = call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris', { + 'list:admin:action:post': '', + 'list:admin:action:subscribe': '', + 'list:admin:action:unsubscribe': '', + 'list:admin:notice:subscribe': '', + 'list:admin:notice:unrecognized': '', + 'list:admin:notice:unsubscribe': '', + 'list:member:digest:footer': '', + 'list:member:digest:header': '', + 'list:member:digest:masthead': '', + 'list:member:regular:footer': 'http://example.org/footer', + 'list:member:regular:header': 'http://example.org/header', + 'list:user:action:confirm': '', + 'list:user:action:unsubscribe': '', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'list:user:notice:hold': '', + 'list:user:notice:no-more-today': '', + 'list:user:notice:post': '', + 'list:user:notice:probe': '', + 'list:user:notice:refuse': '', + 'list:user:notice:welcome': 'http://example.org/welcome', + 'password': 'some password', + 'username': 'anne.person', + }, method='PUT') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:member:digest:footer', 'ant.example.com') + self.assertIsNone(template) + template = manager.raw('list:member:digest:header', 'ant.example.com') + self.assertIsNone(template) + template = manager.raw('list:member:regular:footer', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/footer') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:member:regular:header', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/header') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', 'ant.example.com') + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + + def test_delete_all_uris(self): + manager = getUtility(ITemplateManager) + with transaction(): + manager.set( + 'list:user:notice:welcome', 'ant.example.com', + 'http://example.com/welcome') + manager.set( + 'list:user:notice:goodbye', 'ant.example.com', + 'http://example.com/goodbye', + 'a user', 'the password', + ) + resource, response = call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris', + method='DELETE') + self.assertEqual(response.status, 204) + self.assertIsNone( + manager.raw('list:user:notice:welcome', 'ant.example.com')) + self.assertIsNone( + manager.raw('list:user:notice:goodbye', 'ant.example.com')) + + def test_get_a_url(self): + with transaction(): + getUtility(ITemplateManager).set( + 'list:user:notice:welcome', 'ant.example.com', + 'http://example.com/welcome') + resource, response = call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris' + '/list:user:notice:welcome') + self.assertEqual(response.status, 200) + self.assertEqual(resource, { + 'http_etag': '"36f8bef800cfd278f097c61c5892a34c0650f4aa"', + 'self_link': ('http://localhost:9001/3.1/lists/ant.example.com' + '/uris/list:user:notice:welcome'), + 'uri': 'http://example.com/welcome', + }) + + def test_get_a_bad_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris' + '/list:user:notice:notemplate') + self.assertEqual(cm.exception.code, 404) + + def test_get_unset_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris' + '/list:user:notice:welcome') + self.assertEqual(cm.exception.code, 404) + + def test_patch_url_with_too_many_parameters(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/lists/ant.example.com/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'secret': 'some password', + 'person': 'anne.person', + }, method='PATCH') + self.assertEqual(cm.exception.code, 400) + + def test_deprecated_resources(self): + # This resource does not exist with API 3.0. + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.0/templates/ant@example.com' + '/footer/en') + self.assertEqual(cm.exception.code, 404) diff --git a/src/mailman/rest/tests/test_root.py b/src/mailman/rest/tests/test_root.py index 1e0ff6f28..840212825 100644 --- a/src/mailman/rest/tests/test_root.py +++ b/src/mailman/rest/tests/test_root.py @@ -25,9 +25,12 @@ from base64 import b64encode from httplib2 import Http from mailman.config import config from mailman.core.system import system +from mailman.database.transaction import transaction +from mailman.interfaces.template import ITemplateManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer from urllib.error import HTTPError +from zope.component import getUtility class TestRoot(unittest.TestCase): @@ -167,3 +170,211 @@ class TestRoot(unittest.TestCase): 'chains': ['chain_1', 'chain_2'] }, method='PUT') self.assertEqual(cm.exception.code, 405) + + +class TestSiteTemplates(unittest.TestCase): + """Test /uris""" + + layer = RESTLayer + + def test_no_templates_for_api_30(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/uris') + self.assertEqual(cm.exception.code, 404) + + def test_path_too_long(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/uris/foo/bar') + self.assertEqual(cm.exception.code, 400) + + def test_get_unknown_uri(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/uris/not:a:template') + self.assertEqual(cm.exception.code, 404) + + def test_get_all_uris(self): + manager = getUtility(ITemplateManager) + with transaction(): + manager.set( + 'list:user:notice:welcome', None, + 'http://example.com/welcome') + manager.set( + 'list:user:notice:goodbye', None, + 'http://example.com/goodbye', + 'a user', 'the password', + ) + resource, response = call_api( + 'http://localhost:9001/3.1/uris') + self.assertEqual(response.status, 200) + self.assertEqual(resource['start'], 0) + self.assertEqual(resource['total_size'], 2) + self.assertEqual( + resource['self_link'], + 'http://localhost:9001/3.1/uris') + self.assertEqual(resource['entries'], [ + {'http_etag': '"063fd6635a6035a4b7e939a304fcbd16571aa662"', + 'name': 'list:user:notice:goodbye', + 'password': 'the password', + 'self_link': ('http://localhost:9001/3.1' + '/uris/list:user:notice:goodbye'), + 'uri': 'http://example.com/goodbye', + 'username': 'a user', + }, + {'http_etag': '"5c4ec63b2a0a50f96483ec85b94b80ee092af792"', + 'name': 'list:user:notice:welcome', + 'self_link': ('http://localhost:9001/3.1' + '/uris/list:user:notice:welcome'), + 'uri': 'http://example.com/welcome', + }]) + + def test_patch_uris(self): + resource, response = call_api( + 'http://localhost:9001/3.1/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + }, method='PATCH') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:user:notice:welcome', None) + self.assertEqual(template.uri, 'http://example.org/welcome') + self.assertIsNone(template.username) + self.assertEqual(template.password, '') + template = manager.raw('list:user:notice:goodbye', None) + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertIsNone(template.username) + self.assertEqual(template.password, '') + + def test_patch_uris_with_credentials(self): + resource, response = call_api( + 'http://localhost:9001/3.1/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'password': 'some password', + 'username': 'anne.person', + }, method='PATCH') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:user:notice:welcome', None) + self.assertEqual(template.uri, 'http://example.org/welcome') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', None) + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + + def test_patch_uris_with_partial_credentials(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'username': 'anne.person', + }, method='PATCH') + self.assertEqual(cm.exception.code, 400) + + def test_put_all_uris(self): + resource, response = call_api( + 'http://localhost:9001/3.1/uris', { + 'domain:admin:notice:new-list': '', + 'list:admin:action:post': '', + 'list:admin:action:subscribe': '', + 'list:admin:action:unsubscribe': '', + 'list:admin:notice:subscribe': '', + 'list:admin:notice:unrecognized': '', + 'list:admin:notice:unsubscribe': '', + 'list:member:digest:footer': '', + 'list:member:digest:header': '', + 'list:member:digest:masthead': '', + 'list:member:regular:footer': 'http://example.org/footer', + 'list:member:regular:header': 'http://example.org/header', + 'list:user:action:confirm': '', + 'list:user:action:unsubscribe': '', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'list:user:notice:hold': '', + 'list:user:notice:no-more-today': '', + 'list:user:notice:post': '', + 'list:user:notice:probe': '', + 'list:user:notice:refuse': '', + 'list:user:notice:welcome': 'http://example.org/welcome', + 'password': 'some password', + 'username': 'anne.person', + }, method='PUT') + self.assertEqual(response.status, 204) + manager = getUtility(ITemplateManager) + template = manager.raw('list:member:digest:footer', None) + self.assertIsNone(template) + template = manager.raw('list:member:digest:header', None) + self.assertIsNone(template) + template = manager.raw('list:member:regular:footer', None) + self.assertEqual(template.uri, 'http://example.org/footer') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:member:regular:header', None) + self.assertEqual(template.uri, 'http://example.org/header') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', None) + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + template = manager.raw('list:user:notice:goodbye', None) + self.assertEqual(template.uri, 'http://example.org/goodbye') + self.assertEqual(template.username, 'anne.person') + self.assertEqual(template.password, 'some password') + + def test_delete_all_uris(self): + manager = getUtility(ITemplateManager) + with transaction(): + manager.set( + 'list:user:notice:welcome', None, + 'http://example.com/welcome') + manager.set( + 'list:user:notice:goodbye', None, + 'http://example.com/goodbye', + 'a user', 'the password', + ) + resource, response = call_api( + 'http://localhost:9001/3.1/uris', + method='DELETE') + self.assertEqual(response.status, 204) + self.assertIsNone(manager.raw('list:user:notice:welcome', None)) + self.assertIsNone(manager.raw('list:user:notice:goodbye', None)) + + def test_get_a_url(self): + with transaction(): + getUtility(ITemplateManager).set( + 'list:user:notice:welcome', None, + 'http://example.com/welcome') + resource, response = call_api( + 'http://localhost:9001/3.1/uris/list:user:notice:welcome') + self.assertEqual(response.status, 200) + self.assertEqual(resource, { + 'http_etag': '"86e360d83197561d50826ad6d15e9c30923b82d6"', + 'self_link': ('http://localhost:9001/3.1' + '/uris/list:user:notice:welcome'), + 'uri': 'http://example.com/welcome', + }) + + def test_get_a_bad_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/uris/list:user:notice:notemplate') + self.assertEqual(cm.exception.code, 404) + + def test_get_unset_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/uris/list:user:notice:welcome') + self.assertEqual(cm.exception.code, 404) + + def test_patch_url_with_too_many_parameters(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.1/uris', { + 'list:user:notice:welcome': 'http://example.org/welcome', + 'list:user:notice:goodbye': 'http://example.org/goodbye', + 'secret': 'some password', + 'person': 'anne.person', + }, method='PATCH') + self.assertEqual(cm.exception.code, 400) diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index 9335480bf..e76c082df 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -37,6 +37,7 @@ class TestSystemConfiguration(unittest.TestCase): self.assertIn('http_etag', json) del json['http_etag'] self.assertEqual(json, dict( + cache_life='7d', default_language='en', email_commands_max_lines='10', filtered_messages_are_preservable='no', diff --git a/src/mailman/rest/uris.py b/src/mailman/rest/uris.py new file mode 100644 index 000000000..2fc062262 --- /dev/null +++ b/src/mailman/rest/uris.py @@ -0,0 +1,203 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""URI templates.""" + +from mailman import public +from mailman.interfaces.template import ALL_TEMPLATES, ITemplateManager +from mailman.rest.helpers import ( + CollectionMixin, bad_request, etag, no_content, not_found, okay) +from mailman.rest.validator import Validator +from operator import attrgetter +from zope.component import getUtility + + +class _URIBase(CollectionMixin): + def __init__(self, context): + self._context = context + + def _resource_as_dict(self, template): + resource = dict( + uri=template.uri, + name=template.name, + self_link=self.api.path_to('{}/uris/{}'.format( + self._prefix, template.name)), + ) + if template.username is not None and template.password is not None: + resource['username'] = template.username + resource['password'] = template.password + return resource + + def _get_collection(self, request): + manager = getUtility(ITemplateManager) + collection = [] + for uri in self.URIs: + template = manager.raw(uri, self._raw_context) + if template is not None: + collection.append(template) + return sorted(collection, key=attrgetter('name')) + + def on_get(self, request, response): + resource = self._make_collection(request) + resource['self_link'] = self.api.path_to( + '{}/uris'.format(self._prefix)) + okay(response, etag(resource)) + + def _patch_put(self, request, response, is_optional): + kws = {uri: str for uri in self.URIs} + optionals = ['username', 'password'] + if is_optional: + optionals.extend(self.URIs) + # When PATCHing or PUTing all uris, a single optional + # username/password applies to them all. + kws['username'] = str + kws['password'] = str + kws['_optional'] = optionals + try: + arguments = Validator(**kws)(request) + except ValueError as error: + bad_request(response, str(error)) + return + username = arguments.pop('username', None) + password = arguments.pop('password', None) + if not username and not password: + # Normalize arguments. + set_kws = {} + elif username and password: + # It's fine if both are specified. + set_kws = dict(username=username, password=password) + else: + bad_request(response, + 'Specify both username and password, or neither') + return + manager = getUtility(ITemplateManager) + for key, value in arguments.items(): + if len(value) == 0: + # The empty string is equivalent to DELETE. Yeah, this isn't + # very RESTful, but practicality beats purity. + manager.delete(key, self._raw_context) + else: + manager.set(key, self._raw_context, value, **set_kws) + no_content(response) + + def on_put(self, request, response): + self._patch_put(request, response, is_optional=False) + + def on_patch(self, request, response): + self._patch_put(request, response, is_optional=True) + + def on_delete(self, request, response): + manager = getUtility(ITemplateManager) + for uri in self.URIs: + manager.delete(uri, self._raw_context) + no_content(response) + + +class _ListURIBase(_URIBase): + def __init__(self, context): + super().__init__(context) + self._raw_context = context.list_id + self._prefix = 'lists/{}'.format(context.list_id) + + +@public +class AllListURIs(_ListURIBase): + URIs = [name for name in ALL_TEMPLATES if name.startswith('list:')] + + def __init__(self, context): + super().__init__(context) + + +@public +class AListURI(_ListURIBase): + def __init__(self, context, template): + super().__init__(context) + self.URIs = [template] + self._template = template + + def on_get(self, request, response): + template = getUtility(ITemplateManager).raw( + self._template, self._raw_context) + if template is None: + not_found(response) + else: + resource = dict(uri=template.uri) + resource['self_link'] = self.api.path_to( + '{}/uris/{}'.format(self._prefix, self._template)) + okay(response, etag(resource)) + + +class _DomainURIBase(_URIBase): + def __init__(self, context): + super().__init__(context) + self._raw_context = context.mail_host + self._prefix = 'domains/{}'.format(context.mail_host) + + +@public +class AllDomainURIs(_DomainURIBase): + URIs = [name for name in ALL_TEMPLATES + if name.startswith('list:') or name.startswith('domain:')] + + +@public +class ADomainURI(_DomainURIBase): + def __init__(self, context, template): + super().__init__(context) + self.URIs = [template] + self._template = template + + def on_get(self, request, response): + template = getUtility(ITemplateManager).raw( + self._template, self._raw_context) + if template is None: + not_found(response) + else: + resource = dict(uri=template.uri) + resource['self_link'] = self.api.path_to( + '{}/uris/{}'.format(self._prefix, self._template)) + okay(response, etag(resource)) + + +class _SiteURIBase(_URIBase): + def __init__(self): + super().__init__(None) + self._raw_context = None + self._prefix = '' + + +@public +class AllSiteURIs(_SiteURIBase): + URIs = [name for name in ALL_TEMPLATES] + + +@public +class ASiteURI(_SiteURIBase): + def __init__(self, template): + super().__init__() + self.URIs = [template] + self._template = template + + def on_get(self, request, response): + template = getUtility(ITemplateManager).raw(self._template, None) + if template is None: + not_found(response) + else: + resource = dict(uri=template.uri) + resource['self_link'] = self.api.path_to( + 'uris/{}'.format(self._template)) + okay(response, etag(resource)) diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py index bcc6b3321..19e8d684a 100644 --- a/src/mailman/rest/validator.py +++ b/src/mailman/rest/validator.py @@ -161,7 +161,8 @@ class Validator: :param obj: The object to update. :type obj: object :param request: The HTTP request. - :raises ValueError: if conversion failed for some attribute. + :raises ValueError: if conversion failed for some attribute, including + if the API version mismatches. """ for key, value in self.__call__(request).items(): self._converters[key].put(obj, key, value) diff --git a/src/mailman/rest/wsgiapp.py b/src/mailman/rest/wsgiapp.py index bfe17af73..f3290416e 100644 --- a/src/mailman/rest/wsgiapp.py +++ b/src/mailman/rest/wsgiapp.py @@ -200,7 +200,8 @@ class RootedAPI(API): # Let Falcon parse the form data into the request object's # .params attribute. self.req_options.auto_parse_form_urlencoded = True - # Don't ignore empty query parameters. + # Don't ignore empty query parameters, e.g. preserve empty string + # values, which some resources will interpret as a DELETE. self.req_options.keep_blank_qs_values = True # Override the base class implementation to wrap a transactional diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index bbef1f33f..c591c10a9 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -33,10 +33,10 @@ from mailman.core.runner import Runner from mailman.email.message import Message, MultipartDigestMessage from mailman.handlers.decorate import decorate from mailman.interfaces.member import DeliveryMode, DeliveryStatus -from mailman.utilities.i18n import make +from mailman.interfaces.template import ITemplateLoader from mailman.utilities.mailbox import Mailbox -from mailman.utilities.string import oneline, wrap -from urllib.error import URLError +from mailman.utilities.string import expand, oneline, wrap +from zope.component import getUtility log = logging.getLogger('mailman.error') @@ -67,23 +67,16 @@ class Digester: # digest header are separate MIME subobjects. In either case, it's # the first thing in the digest, and we can calculate it now, so go # ahead and add it now. - self._masthead = make('masthead.txt', - mailing_list=mlist, - display_name=mlist.display_name, - got_list_email=mlist.posting_address, - got_listinfo_url=mlist.script_url('listinfo'), - got_request_email=mlist.request_address, - got_owner_email=mlist.owner_address, - ) + template = getUtility(ITemplateLoader).get( + 'list:member:digest:masthead', mlist) + self._masthead = wrap(expand(template, mlist, dict( + # For backward compatibility. + got_list_email=mlist.posting_address, + got_request_email=mlist.request_address, + got_owner_email=mlist.owner_address, + ))) # Set things up for the table of contents. - if mlist.digest_header_uri is not None: - try: - self._header = decorate(mlist, mlist.digest_header_uri) - except URLError: - log.exception( - 'Digest header decorator URI not found ({}): {}'.format( - mlist.fqdn_listname, mlist.digest_header_uri)) - self._header = '' + self._header = decorate('list:member:digest:header', mlist) self._toc = StringIO() print(_("Today's Topics:\n"), file=self._toc) @@ -151,7 +144,7 @@ class MIMEDigester(Digester): masthead['Content-Description'] = self._subject self._message.attach(masthead) # Add the optional digest header. - if mlist.digest_header_uri is not None: + if len(self._header) > 0: header = MIMEText(self._header.encode(self._charset), _charset=self._charset) header['Content-Description'] = _('Digest Header') @@ -186,16 +179,8 @@ class MIMEDigester(Digester): def finish(self): """Finish up the digest, producing the email-ready copy.""" self._message.attach(self._digest_part) - if self._mlist.digest_footer_uri is not None: - try: - footer_text = decorate( - self._mlist, self._mlist.digest_footer_uri) - except URLError: - log.exception( - 'Digest footer decorator URI not found ({0}): {1}'.format( - self._mlist.fqdn_listname, - self._mlist.digest_footer_uri)) - footer_text = '' + footer_text = decorate('list:member:digest:footer', self._mlist) + if len(footer_text) > 0: footer = MIMEText(footer_text.encode(self._charset), _charset=self._charset) footer['Content-Description'] = _('Digest Footer') @@ -220,7 +205,7 @@ class RFC1153Digester(Digester): print(self._masthead, file=self._text) print(file=self._text) # Add the optional digest header. - if mlist.digest_header_uri is not None: + if len(self._header) > 0: print(self._header, file=self._text) print(file=self._text) # Calculate the set of headers we're to keep in the RFC1153 digest. @@ -273,16 +258,8 @@ class RFC1153Digester(Digester): def finish(self): """Finish up the digest, producing the email-ready copy.""" - if self._mlist.digest_footer_uri is not None: - try: - footer_text = decorate( - self._mlist, self._mlist.digest_footer_uri) - except URLError: - log.exception( - 'Digest footer decorator URI not found ({}): {}'.format( - self._mlist.fqdn_listname, - self._mlist.digest_footer_uri)) - footer_text = '' + footer_text = decorate('list:member:digest:footer', self._mlist) + if len(footer_text) > 0: # MAS: There is no real place for the digest_footer in an RFC 1153 # compliant digest, so add it as an additional message with # Subject: Digest Footer diff --git a/src/mailman/runners/docs/digester.rst b/src/mailman/runners/docs/digester.rst index c5b1a0e6e..ea31761a1 100644 --- a/src/mailman/runners/docs/digester.rst +++ b/src/mailman/runners/docs/digester.rst @@ -139,9 +139,8 @@ The MIME digest has lots of good stuff, all contained in the multipart. Send Test mailing list submissions to test@example.com - To subscribe or unsubscribe via the World Wide Web, visit - http://lists.example.com/listinfo/test@example.com - or, via email, send a message with subject or body 'help' to + To subscribe or unsubscribe via email, send a message with subject or + body 'help' to test-request@example.com You can reach the person managing the list at @@ -221,7 +220,6 @@ The MIME digest has lots of good stuff, all contained in the multipart. _______________________________________________ Test mailing list test@example.com - http://lists.example.com/listinfo/test@example.com --===============...==-- @@ -242,9 +240,8 @@ The RFC 1153 contains the digest in a single plain text message. Send Test mailing list submissions to test@example.com - To subscribe or unsubscribe via the World Wide Web, visit - http://lists.example.com/listinfo/test@example.com - or, via email, send a message with subject or body 'help' to + To subscribe or unsubscribe via email, send a message with subject or + body 'help' to test-request@example.com You can reach the person managing the list at @@ -300,7 +297,6 @@ The RFC 1153 contains the digest in a single plain text message. _______________________________________________ Test mailing list test@example.com - http://lists.example.com/listinfo/test@example.com ------------------------------ diff --git a/src/mailman/runners/tests/test_digest.py b/src/mailman/runners/tests/test_digest.py index 0448a7349..6157f500b 100644 --- a/src/mailman/runners/tests/test_digest.py +++ b/src/mailman/runners/tests/test_digest.py @@ -17,6 +17,7 @@ """Test the digest runner.""" +import os import unittest from email.iterators import _structure as structure @@ -26,6 +27,7 @@ from mailman.app.lifecycle import create_list from mailman.config import config from mailman.email.message import Message from mailman.interfaces.member import DeliveryMode +from mailman.interfaces.template import ITemplateManager from mailman.runners.digest import DigestRunner from mailman.testing.helpers import ( LogFileMark, digest_mbox, get_queue_messages, make_digest_messages, @@ -34,6 +36,8 @@ from mailman.testing.helpers import ( subscribe) from mailman.testing.layers import ConfigLayer from string import Template +from tempfile import TemporaryDirectory +from zope.component import getUtility class TestDigest(unittest.TestCase): @@ -220,6 +224,29 @@ class TestI18nDigest(unittest.TestCase): self._mlist.digest_size_threshold = 0 self._process = config.handlers['to-digest'].process self._runner = make_testable_runner(DigestRunner) + # Add a French version of the digest masthead. + tempdir = TemporaryDirectory() + self.addCleanup(tempdir.cleanup) + french_path = os.path.join(tempdir.name, 'fr', 'masthead.txt') + os.makedirs(os.path.dirname(french_path)) + with open(french_path, 'w', encoding='utf-8') as fp: + print("""\ +Envoyez vos messages pour la liste $display_name à +\t$got_list_email + +Pour vous (dés)abonner par courriel, envoyez un message avec « help » dans +le corps ou dans le sujet à +\t$got_request_email + +Vous pouvez contacter l'administrateur de la liste à l'adresse +\t$got_owner_email + +Si vous répondez, n'oubliez pas de changer l'objet du message afin +qu'il soit plus spécifique que « Re: Contenu du groupe de $display_name... +""", file=fp) + getUtility(ITemplateManager).set( + 'list:member:digest:masthead', self._mlist.list_id, + 'file:///{}/$language/masthead.txt'.format(tempdir.name)) def test_multilingual_digest(self): # When messages come in with a content-type character set different diff --git a/src/mailman/styles/base.py b/src/mailman/styles/base.py index e6f8e182f..6620e85f2 100644 --- a/src/mailman/styles/base.py +++ b/src/mailman/styles/base.py @@ -82,9 +82,6 @@ class BasicOperation: mlist.mime_is_default_digest = False mlist.digest_size_threshold = 30 # KB mlist.digest_send_periodic = True - mlist.digest_header_uri = None - mlist.digest_footer_uri = ( - 'mailman:///$listname/$language/footer-generic.txt') mlist.digest_volume_frequency = DigestFrequency.monthly mlist.next_digest_number = 1 # NNTP gateway @@ -127,9 +124,6 @@ class BasicOperation: # is that they will get all messages, and they will not have an entry # in this dictionary. mlist.topics_userinterest = {} - # Other - mlist.header_uri = None - mlist.footer_uri = 'mailman:///$listname/$language/footer-generic.txt' # scrub regular delivery mlist.scrub_nondigest = False @@ -204,8 +198,6 @@ class Announcement: mlist.send_welcome_message = True mlist.send_goodbye_message = True mlist.anonymous_list = False - mlist.welcome_message_uri = 'mailman:///welcome.txt' - mlist.goodbye_message_uri = '' @public @@ -219,8 +211,6 @@ class Discussion: mlist.send_welcome_message = True mlist.send_goodbye_message = True mlist.anonymous_list = False - mlist.welcome_message_uri = 'mailman:///welcome.txt' - mlist.goodbye_message_uri = '' @public diff --git a/src/mailman/templates/en/adminaddrchgack.txt b/src/mailman/templates/en/adminaddrchgack.txt deleted file mode 100644 index 1b66a2fcf..000000000 --- a/src/mailman/templates/en/adminaddrchgack.txt +++ /dev/null @@ -1,4 +0,0 @@ -Address for member $name has been successfully changed -from $oldaddr to $newaddr for list $listname. - - diff --git a/src/mailman/templates/en/admindbdetails.html b/src/mailman/templates/en/admindbdetails.html deleted file mode 100644 index d97fcec11..000000000 --- a/src/mailman/templates/en/admindbdetails.html +++ /dev/null @@ -1,65 +0,0 @@ -The administrative requests are displayed in one of two ways, on a summary page, and on a details -page. The summary page contains pending subscription and -unsubscription requests, as well as postings being held for your -approval, grouped by sender email address. The details page contains -a more detailed view of each held message, including the all the -message's headers and an excerpt of the message body. - -

On all the pages, the following actions are available: - -

    -
  • Defer -- Defer your decision until later. No action is - taken now for this pending administrative request, but for held - postings, you can still forward or preserve the message (see - below). - -
  • Accept/Approve -- Accept the message, sending it on to the list. - For membership requests, approve the change in membership status. - -
  • Reject -- Reject the message, sending a rejection notice to - the sender, and discarding the original message. For membership - requests, reject the change in membership status. In either case, - you should add a reason for the rejection in the accompanying text - box. - -
  • Discard -- Throw away the original message, without sending - a rejection notice. For membership requests, this simply discards - the request without notice to the person making the request. This - is usually the action you want to take for known spam. -
- -

For held messages, turn on the Preserve option if you want -to save a copy of the message for the site administrator. This is -useful for abusive messages that you want to discard, but need to keep -a record of for later inspection. - -

Turn on the Forward to option, and fill in the forwarding -address if you want to forward the message to someone else not on the -list. To edit a held message before it is sent on to the list, you -should forward the message to yourself (or the list owners), and -discard the original message. Then, when the message shows up in your -mailbox, make your edits and resend the message to the list, including -an Approved: header with the list password as its value. It -is proper netiquette in this case to include a note in the resent -message, explaining that you have modified the text. - -

If the sender is a list member who is being moderated, you can -optionally clear their moderation flag. This is useful when your list -is configured to put new members on probation, and you've decided that -this member can be trusted to post to the list without approval. - -

If the sender is not a list member, you can add the email address to -a sender filter. Sender filters are described on the sender filter privacy page, and may be one of -auto-accept (Accepts), auto-hold (Holds), -auto-reject (Rejects), or auto-discard (Discards). This -option will not be available if the address is already on one of the -sender filters. - -

When you're finished, click on the Submit All Data button -at the top or bottom of the page. This button will submit all -selected actions for all administrative requests that you've made a -decision for. - -

Return to the summary page. diff --git a/src/mailman/templates/en/admindbpreamble.html b/src/mailman/templates/en/admindbpreamble.html deleted file mode 100644 index 659b77e72..000000000 --- a/src/mailman/templates/en/admindbpreamble.html +++ /dev/null @@ -1,10 +0,0 @@ -This page contains a subset of the %(listname)s mailing list -postings that are being held for your approval. It currently shows -%(description)s - -

For each administrative request, please select the action to take, -clicking on the Submit All Data when finished. More detailed -instructions are available here. - -

You can also view a summary of all -pending requests. diff --git a/src/mailman/templates/en/admindbsummary.html b/src/mailman/templates/en/admindbsummary.html deleted file mode 100644 index 20ffef584..000000000 --- a/src/mailman/templates/en/admindbsummary.html +++ /dev/null @@ -1,14 +0,0 @@ -This page contains a summary of the current set of administrative -requests requiring your approval for the -%(listname)s mailing list. -First, you will find the list of pending -subscription and unsubscription requests, if any, followed by any -postings being held for your approval. - -

For each administrative request, please select the action to take, -clicking on the Submit All Data button when finished. -More detailed instructions are also -available. - -

You can also view the details of all -held postings. diff --git a/src/mailman/templates/en/adminsubscribeack.txt b/src/mailman/templates/en/adminsubscribeack.txt deleted file mode 100644 index 5d5f36417..000000000 --- a/src/mailman/templates/en/adminsubscribeack.txt +++ /dev/null @@ -1 +0,0 @@ -$member has been successfully subscribed to $listname. diff --git a/src/mailman/templates/en/adminunsubscribeack.txt b/src/mailman/templates/en/adminunsubscribeack.txt deleted file mode 100644 index a6ed99f59..000000000 --- a/src/mailman/templates/en/adminunsubscribeack.txt +++ /dev/null @@ -1 +0,0 @@ -$member has been removed from $listname. diff --git a/src/mailman/templates/en/admlogin.html b/src/mailman/templates/en/admlogin.html deleted file mode 100644 index 03f763b95..000000000 --- a/src/mailman/templates/en/admlogin.html +++ /dev/null @@ -1,39 +0,0 @@ - - - %(listname)s %(who)s Authentication - - -

-%(message)s - - - - - - - - - - - -
- %(listname)s %(who)s - Authentication -
List %(who)s Password:
-
-

Important: From this point on, you - must have cookies enabled in your browser, otherwise no - administrative changes will take effect. - -

Session cookies are used in Mailman's - administrative interface so that you don't need to - re-authenticate with every administrative operation. This - cookie will expire automatically when you exit your browser, or - you can explicitly expire the cookie by hitting the - Logout link under Other Administrative - Activities (which you'll see once you successfully log in). -

- - diff --git a/src/mailman/templates/en/approve.txt b/src/mailman/templates/en/approve.txt deleted file mode 100644 index 10ab69356..000000000 --- a/src/mailman/templates/en/approve.txt +++ /dev/null @@ -1,15 +0,0 @@ -Your request to $requestaddr: - - $cmd - -has been forwarded to the person running the list. - -This is probably because you are trying to subscribe to a 'closed' -list. - -You will receive email notification of the list owner's decision about -your subscription request. - -Any questions about the list owner's policy should be directed to: - - $adminaddr diff --git a/src/mailman/templates/en/article.html b/src/mailman/templates/en/article.html deleted file mode 100644 index 9ba25fe4e..000000000 --- a/src/mailman/templates/en/article.html +++ /dev/null @@ -1,50 +0,0 @@ - - - - $title - - - - - $encoding - $prev - $next - - -

$subject_html

- $author_html - $email_html -
- $datestr_html -

-
- -$body - -
-

- -
-More information about the $listname -mailing list
- diff --git a/src/mailman/templates/en/bounce.txt b/src/mailman/templates/en/bounce.txt deleted file mode 100644 index 8e02cc7a5..000000000 --- a/src/mailman/templates/en/bounce.txt +++ /dev/null @@ -1,13 +0,0 @@ -This is a Mailman mailing list bounce action notice: - - List: %(listname)s - Member: %(addr)s - Action: Subscription %(negative)s%(did)s. - Reason: Excessive or fatal bounces. - %(but)s - -%(reenable)s -The triggering bounce notice is attached below. - -Questions? -Contact the Mailman site administrator at %(owneraddr)s. diff --git a/src/mailman/templates/en/checkdbs.txt b/src/mailman/templates/en/checkdbs.txt deleted file mode 100644 index d53925a4d..000000000 --- a/src/mailman/templates/en/checkdbs.txt +++ /dev/null @@ -1,7 +0,0 @@ -The %(real_name)s@%(host_name)s mailing list has %(count)d request(s) -waiting for your consideration at: - - %(adminDB)s - -Please attend to this at your earliest convenience. This notice of -pending requests, if any, will be sent out daily. diff --git a/src/mailman/templates/en/confirm.txt b/src/mailman/templates/en/confirm.txt deleted file mode 100644 index 7c8bee75f..000000000 --- a/src/mailman/templates/en/confirm.txt +++ /dev/null @@ -1,17 +0,0 @@ -Email Address Registration Confirmation - -Hello, this is the GNU Mailman server at ${domain_name}. - -We have received a registration request for the email address - - $email_address - -Before you can start using GNU Mailman at this site, you must first confirm -that this is your email address. You can do this by replying to this message, -keeping the Subject header intact. - -If you do not wish to register this email address simply disregard this -message. If you think you are being maliciously subscribed to the list, or -have any other questions, you may contact - - $contact_address diff --git a/src/mailman/templates/en/convert.txt b/src/mailman/templates/en/convert.txt deleted file mode 100644 index ae17a79e0..000000000 --- a/src/mailman/templates/en/convert.txt +++ /dev/null @@ -1,34 +0,0 @@ -The %(listname)s mailing list has just undergone a big change. It is -running on a new mailing list package called "Mailman". This will -hopefully solve a lot of problems that administering this list has -presented. - -How does this affect you? - -1) Mail intended for the whole list should be sent to: %(listaddr)s. - -2) You have been given an arbitrary password to prevent others from -unsubscribing you without your knowledge. It will be mailed to you in -a separate email, which you may have already received. Don't worry if -you forget this password; a reminder will be sent to you via email -every month. - -3) If you have World Wide Web access, you can use it any time to -unsubscribe from this list, to switch to and from digest mode, to -check back issues of the list (which will be available after the list -has been getting posts for a day or so), etc. The Web address for -these resources is: - - %(listinfo_url)s - -4) If you do not have WWW access, you can do these same things via -email. Send mail to %(requestaddr)s with a subject or body containing -just the word "help" (without the quotes). You will receive an -automated reply giving you further directions. - -Please address any questions or problems with this new setup to: -%(adminaddr)s. - -This message was auto-generated by Mailman %(version)s. For more -information on the Mailman software, visit the Mailman homepage at -http://www.list.org/ diff --git a/src/mailman/templates/en/cronpass.txt b/src/mailman/templates/en/cronpass.txt deleted file mode 100644 index 52ce5ea6c..000000000 --- a/src/mailman/templates/en/cronpass.txt +++ /dev/null @@ -1,19 +0,0 @@ -This is a reminder, sent out once a month, about your %(hostname)s -mailing list memberships. It includes your subscription info and how -to use it to change it or unsubscribe from a list. - -You can visit the URLs to change your membership status or -configuration, including unsubscribing, setting digest-style delivery -or disabling delivery altogether (e.g., for a vacation), and so on. - -In addition to the URL interfaces, you can also use email to make such -changes. For more info, send a message to the '-request' address of -the list (for example, %(exreq)s) containing just the word 'help' in -the message body, and an email message will be sent to you with -instructions. - -If you have questions, problems, comments, etc, send them to -%(owner)s. Thanks! - -Passwords for %(useraddr)s: - diff --git a/src/mailman/templates/en/disabled.txt b/src/mailman/templates/en/disabled.txt deleted file mode 100644 index 54998a83b..000000000 --- a/src/mailman/templates/en/disabled.txt +++ /dev/null @@ -1,25 +0,0 @@ -Your membership in the mailing list %(listname)s has been disabled -%(reason)s. You will not get any more messages from this -list until you re-enable your membership. You will receive -%(noticesleft)s more reminders like this before your membership in the -list is deleted. - -To re-enable your membership, you can simply respond to this message -(leaving the Subject: line intact), or visit the confirmation page at - - %(confirmurl)s - -You can also visit your membership page at - - %(optionsurl)s - -On your membership page, you can change various delivery options such -as your email address and whether you get digests or not. As a -reminder, your membership password is - - %(password)s - -If you have any questions or problems, you can contact the list owner -at - - %(owneraddr)s diff --git a/src/mailman/templates/en/domain:admin:notice:new-list.txt b/src/mailman/templates/en/domain:admin:notice:new-list.txt new file mode 100644 index 000000000..8320c27d0 --- /dev/null +++ b/src/mailman/templates/en/domain:admin:notice:new-list.txt @@ -0,0 +1,10 @@ +The mailing list '$listname' has just been created for you. The +following is some basic information about your mailing list. + +There is an email-based interface for users (not administrators) of your list; +you can get info about using it by sending a message with just the word 'help' +as subject or in the body, to: + + $request_email + +Please address all questions to $site_email. diff --git a/src/mailman/templates/en/emptyarchive.html b/src/mailman/templates/en/emptyarchive.html deleted file mode 100644 index 2f10766ce..000000000 --- a/src/mailman/templates/en/emptyarchive.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - The %(listname)s Archives - - - -

The %(listname)s Archives

-

- No messages have been posted to this list yet, so the archives are - currently empty. You can get more information - about this list. -

- - diff --git a/src/mailman/templates/en/footer-generic.txt b/src/mailman/templates/en/footer-generic.txt deleted file mode 100644 index d31e885f0..000000000 --- a/src/mailman/templates/en/footer-generic.txt +++ /dev/null @@ -1,4 +0,0 @@ -_______________________________________________ -$display_name mailing list -$fqdn_listname -${listinfo_uri} diff --git a/src/mailman/templates/en/headfoot.html b/src/mailman/templates/en/headfoot.html deleted file mode 100644 index b2caf10f6..000000000 --- a/src/mailman/templates/en/headfoot.html +++ /dev/null @@ -1,28 +0,0 @@ -This text can include -Python -format strings which are resolved against list attributes. The -list of substitutions allowed are: - -
    -
  • real_name - The "pretty" name of the list; usually - the list name with capitalization. - -
  • list_name - The name by which the list is - identified in URLs, where case is significant. - -
  • host_name - The fully qualified domain name - that the list server runs on. - -
  • web_page_url - The base URL for Mailman. This - can be appended with, - e.g. listinfo/%(list_name)s to yield the - listinfo page for the mailing list. - -
  • description - The brief description of the - mailing list. - -
  • info - The full description of the mailing - list. - -
  • cgiext - The extension added to CGI scripts. -
diff --git a/src/mailman/templates/en/help.txt b/src/mailman/templates/en/help.txt index 654eda315..1dc31532c 100644 --- a/src/mailman/templates/en/help.txt +++ b/src/mailman/templates/en/help.txt @@ -1,22 +1,11 @@ -Help for %(listname)s mailing list: +Help for $listname mailing list -This is email command help for version %(version)s of the "Mailman" -list manager. The following describes commands you can send to get -information about and control your subscription to Mailman lists at -this site. A command can be in the subject line or in the body of the -message. +This is email command 'help' for version $version of the GNU Mailman list +manager at $domain. The following describes commands you can send to get +information about and control your subscription to Mailman lists at this site. +A command can be in the Subject line or in the body of the message. -Note that much of the following can also be accomplished via the World -Wide Web, at: - - %(listinfo_url)s - -In particular, you can use the Web site to have your password sent to -your delivery address. - -List specific commands (subscribe, who, etc) should be sent to the -*-request address for the particular list, e.g. for the 'mailman' -list, use 'mailman-request@...'. +Commands should be sent to the ${listname}-request@${domain} address. About the descriptions - words in "<>"s signify REQUIRED items and words in "[]" denote OPTIONAL items. Do not include the "<>"s or @@ -24,10 +13,8 @@ words in "[]" denote OPTIONAL items. Do not include the "<>"s or The following commands are valid: - %(commands)s - -Commands should be sent to %(requestaddr)s + $commands -Questions and concerns for the attention of a person should be sent to +Questions and concerns for the attention of a person should be sent to: - %(adminaddr)s + $administrator diff --git a/src/mailman/templates/en/invite.txt b/src/mailman/templates/en/invite.txt index 337ac2ddc..a9494a1a4 100644 --- a/src/mailman/templates/en/invite.txt +++ b/src/mailman/templates/en/invite.txt @@ -3,10 +3,6 @@ mailing list at $hostname by the $listname mailing list owner. You may accept the invitation by simply replying to this message, keeping the Subject: header intact. -You can also visit this web page: - - $confirmurl - Or you should include the following line -- and only the following line -- in a message to $requestaddr: diff --git a/src/mailman/templates/en/list:admin:action:post.txt b/src/mailman/templates/en/list:admin:action:post.txt new file mode 100644 index 000000000..46e6a0fdf --- /dev/null +++ b/src/mailman/templates/en/list:admin:action:post.txt @@ -0,0 +1,13 @@ +As list administrator, your authorization is requested for the +following mailing list posting: + + List: $listname + From: $sender_email + Subject: $subject + +The message is being held because: + +$reasons + +At your convenience, visit your dashboard to approve or deny the +request. diff --git a/src/mailman/templates/en/list:admin:action:subscribe.txt b/src/mailman/templates/en/list:admin:action:subscribe.txt new file mode 100644 index 000000000..7dec73864 --- /dev/null +++ b/src/mailman/templates/en/list:admin:action:subscribe.txt @@ -0,0 +1,5 @@ +Your authorization is required for a mailing list subscription request +approval: + + For: $member + List: $listname diff --git a/src/mailman/templates/en/list:admin:action:unsubscribe.txt b/src/mailman/templates/en/list:admin:action:unsubscribe.txt new file mode 100644 index 000000000..e4825561a --- /dev/null +++ b/src/mailman/templates/en/list:admin:action:unsubscribe.txt @@ -0,0 +1,5 @@ +Your authorization is required for a mailing list unsubscription +request approval: + + For: $member + List: $listname diff --git a/src/mailman/templates/en/list:admin:notice:subscribe.txt b/src/mailman/templates/en/list:admin:notice:subscribe.txt new file mode 100644 index 000000000..455fbc072 --- /dev/null +++ b/src/mailman/templates/en/list:admin:notice:subscribe.txt @@ -0,0 +1 @@ +$member has been successfully subscribed to $display_name. diff --git a/src/mailman/templates/en/list:admin:notice:unrecognized.txt b/src/mailman/templates/en/list:admin:notice:unrecognized.txt new file mode 100644 index 000000000..e520888e2 --- /dev/null +++ b/src/mailman/templates/en/list:admin:notice:unrecognized.txt @@ -0,0 +1,4 @@ +The attached message was received as a bounce, but either the bounce format +was not recognized, or no member addresses could be extracted from it. This +mailing list has been configured to send all unrecognized bounce messages to +the list administrator(s). diff --git a/src/mailman/templates/en/list:admin:notice:unsubscribe.txt b/src/mailman/templates/en/list:admin:notice:unsubscribe.txt new file mode 100644 index 000000000..98fbc30e8 --- /dev/null +++ b/src/mailman/templates/en/list:admin:notice:unsubscribe.txt @@ -0,0 +1 @@ +$member has been removed from $display_name. diff --git a/src/mailman/templates/en/list:member:digest:masthead.txt b/src/mailman/templates/en/list:member:digest:masthead.txt new file mode 100644 index 000000000..0d890d4e3 --- /dev/null +++ b/src/mailman/templates/en/list:member:digest:masthead.txt @@ -0,0 +1,12 @@ +Send $display_name mailing list submissions to + $listname + +To subscribe or unsubscribe via email, send a message with subject or body +'help' to + $request_email + +You can reach the person managing the list at + $owner_email + +When replying, please edit your Subject line so it is more specific than +"Re: Contents of $display_name digest..." diff --git a/src/mailman/templates/en/list:member:generic:footer.txt b/src/mailman/templates/en/list:member:generic:footer.txt new file mode 100644 index 000000000..5985feb6e --- /dev/null +++ b/src/mailman/templates/en/list:member:generic:footer.txt @@ -0,0 +1,3 @@ +_______________________________________________ +$display_name mailing list +$listname diff --git a/src/mailman/templates/en/list:user:action:confirm.txt b/src/mailman/templates/en/list:user:action:confirm.txt new file mode 100644 index 000000000..ce384fea2 --- /dev/null +++ b/src/mailman/templates/en/list:user:action:confirm.txt @@ -0,0 +1,17 @@ +Email Address Registration Confirmation + +Hello, this is the GNU Mailman server at $domain. + +We have received a registration request for the email address + + $user_email + +Before you can start using GNU Mailman at this site, you must first confirm +that this is your email address. You can do this by replying to this message, +keeping the Subject header intact. + +If you do not wish to register this email address simply disregard this +message. If you think you are being maliciously subscribed to the list, or +have any other questions, you may contact + + $owner_email diff --git a/src/mailman/templates/en/list:user:action:unsubscribe.txt b/src/mailman/templates/en/list:user:action:unsubscribe.txt new file mode 100644 index 000000000..97ebd30b2 --- /dev/null +++ b/src/mailman/templates/en/list:user:action:unsubscribe.txt @@ -0,0 +1,19 @@ +Mailing list removal confirmation notice for mailing list $listname + +We have received a request for the removal of your email address, "${email}" +from the $listaddr mailing list. To confirm that you want to be removed from +this mailing list, simply reply to this message, keeping the Subject header +intact. + +Or include the following line -- and only the following line -- in a +message to $requestaddr: + + confirm $cookie + +Note that simply sending a reply to this message should work from most mail +readers, since that usually leaves the Subject line in the right form +(additional "Re:" text in the Subject is okay). + +If you do not wish to be removed from this list, please simply disregard this +message. If you think you are being maliciously removed from the list, or +have any other questions, send them to $listadmin diff --git a/src/mailman/templates/en/list:user:notice:hold.txt b/src/mailman/templates/en/list:user:notice:hold.txt new file mode 100644 index 000000000..ee769c8ae --- /dev/null +++ b/src/mailman/templates/en/list:user:notice:hold.txt @@ -0,0 +1,12 @@ +Your mail to '$listname' with the subject + + $subject + +Is being held until the list moderator can review it for approval. + +The message is being held because: + +$reasons + +Either the message will get posted to the list, or you will receive +notification of the moderator's decision. diff --git a/src/mailman/templates/en/list:user:notice:no-more-today.txt b/src/mailman/templates/en/list:user:notice:no-more-today.txt new file mode 100644 index 000000000..f41751b74 --- /dev/null +++ b/src/mailman/templates/en/list:user:notice:no-more-today.txt @@ -0,0 +1,9 @@ +We have received a message from your address <$sender_email> requesting an +automated response from the $listname mailing list. + +The number we have seen today: $count. In order to avoid problems such as +mail loops between email robots, we will not be sending you any further +responses today. Please try again tomorrow. + +If you believe this message is in error, or if you have any questions, please +contact the list owner at $owner_email. diff --git a/src/mailman/templates/en/list:user:notice:post.txt b/src/mailman/templates/en/list:user:notice:post.txt new file mode 100644 index 000000000..cc3b5af4a --- /dev/null +++ b/src/mailman/templates/en/list:user:notice:post.txt @@ -0,0 +1,5 @@ +Your message entitled + + $subject + +was successfully received by the $display_name mailing list. diff --git a/src/mailman/templates/en/list:user:notice:probe.txt b/src/mailman/templates/en/list:user:notice:probe.txt new file mode 100644 index 000000000..dfcb70229 --- /dev/null +++ b/src/mailman/templates/en/list:user:notice:probe.txt @@ -0,0 +1,14 @@ +This is a probe message. You can ignore this message. + +The $listname mailing list has received a number of bounces from you, +indicating that there may be a problem delivering messages to $sender_email. A +sample is attached below. Please examine this message to make sure there are +no problems with your email address. You may want to check with your mail +administrator for more help. + +You don't need to do anything to remain an enabled member of the mailing list. + +If you have any questions or problems, you can contact the mailing list owner +at + + $owner_email diff --git a/src/mailman/templates/en/list:user:notice:refuse.txt b/src/mailman/templates/en/list:user:notice:refuse.txt new file mode 100644 index 000000000..ca43893a0 --- /dev/null +++ b/src/mailman/templates/en/list:user:notice:refuse.txt @@ -0,0 +1,13 @@ +Your request to the $listname mailing list + + $request + +has been rejected by the list moderator. The moderator gave the +following reason for rejecting your request: + +"$reason" + +Any questions or comments should be directed to the list administrator +at: + + $owner_email diff --git a/src/mailman/templates/en/list:user:notice:welcome.txt b/src/mailman/templates/en/list:user:notice:welcome.txt new file mode 100644 index 000000000..d3d557179 --- /dev/null +++ b/src/mailman/templates/en/list:user:notice:welcome.txt @@ -0,0 +1,15 @@ +Welcome to the "$display_name" mailing list! + +To post to this list, send your email to: + + $listname + +You can make such adjustments via email by sending a message to: + + $request_email + +with the word 'help' in the subject or body (don't include the quotes), and +you will get back a message with instructions. You will need your password to +change your options, but for security purposes, this email is not included +here. There is also a button on your options page that will send your current +password to you. diff --git a/src/mailman/templates/en/listinfo.html b/src/mailman/templates/en/listinfo.html deleted file mode 100644 index 0f0b5e614..000000000 --- a/src/mailman/templates/en/listinfo.html +++ /dev/null @@ -1,143 +0,0 @@ - - - <MM-List-Name> Info Page - - - - -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -- - -
-

  -

- About - - - - -
-

-

To see the collection of prior postings to the list, - visit the - Archives. - -

-
- Using -
- To post a message to all the list members, send email to - . - -

You can subscribe to the list, or change your existing - subscription, in the sections below. -

- Subscribing to -
-

- Subscribe to by filling out the following - form. - -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Your email address: -  
    Your name (optional): 
    You may enter a - privacy password below. This provides only mild security, - but should prevent others from messing with your - subscription. Do not use a valuable password as - it will occasionally be emailed back to you in cleartext. - -

    If you choose not to enter a password, one will be - automatically generated for you, and it will be sent to - you once you've confirmed your subscription. You can - always request a mail-back of your password when you edit - your personal options. - -

    Pick a password: 
    Reenter password to confirm: 
    Which language do you prefer to display your messages?  
    Would you like to receive list mail batched in a daily - digest? - No - Yes -
    -

    -
    - -
-
- - Subscribers -
- - - -

- - - -

- - - diff --git a/src/mailman/templates/en/masthead.txt b/src/mailman/templates/en/masthead.txt deleted file mode 100644 index 5d4cc9696..000000000 --- a/src/mailman/templates/en/masthead.txt +++ /dev/null @@ -1,13 +0,0 @@ -Send $display_name mailing list submissions to - $got_list_email - -To subscribe or unsubscribe via the World Wide Web, visit - $got_listinfo_url -or, via email, send a message with subject or body 'help' to - $got_request_email - -You can reach the person managing the list at - $got_owner_email - -When replying, please edit your Subject line so it is more specific than -"Re: Contents of $display_name digest..." diff --git a/src/mailman/templates/en/newlist.txt b/src/mailman/templates/en/newlist.txt deleted file mode 100644 index f7b164eb4..000000000 --- a/src/mailman/templates/en/newlist.txt +++ /dev/null @@ -1,18 +0,0 @@ -The mailing list '$listname' has just been created for you. The -following is some basic information about your mailing list. - -You can configure your mailing list at the following web page: - - $admin_url - -The web page for users of your mailing list is: - - $listinfo_url - -There is also an email-based interface for users (not administrators) of your -list; you can get info about using it by sending a message with just the word -'help' as subject or in the body, to: - - $requestaddr - -Please address all questions to $siteowner. diff --git a/src/mailman/templates/en/nomoretoday.txt b/src/mailman/templates/en/nomoretoday.txt deleted file mode 100644 index 51e571cf2..000000000 --- a/src/mailman/templates/en/nomoretoday.txt +++ /dev/null @@ -1,9 +0,0 @@ -We have received a message from your address <$sender> requesting an automated -response from the $listname mailing list. - -The number we have seen today: $count. In order to avoid problems such as -mail loops between email robots, we will not be sending you any further -responses today. Please try again tomorrow. - -If you believe this message is in error, or if you have any questions, please -contact the list owner at $owneremail. diff --git a/src/mailman/templates/en/options.html b/src/mailman/templates/en/options.html deleted file mode 100644 index 8213b1f4b..000000000 --- a/src/mailman/templates/en/options.html +++ /dev/null @@ -1,316 +0,0 @@ - - - - <MM-Presentable-User> membership configuration for <MM-List-Name> - - - - - -
- - mailing list membership configuration for - -
-

- - - - - -
- 's subscription status, - password, and options for the mailing list. -
- - - - -

-

- - -

- - - - - - - - -
- - Changing your membership information -
You can change the address that you are subscribed - to the mailing list with by entering the new address in the - fields below. Note that a confirmation email will be sent to - the new address, and the change must be confirmed before it is - processed. - -

Confirmations time out after about . - -

You can also optionally set or change your real name - (i.e. John Smith). - -

If you want to make the membership changes for all the - lists that you are subscribed to at , turn on the - Change globally check box. - -

- - - - - - - -
New address:
Again to - confirm:
-
- - - - -
Your name - (optional):
-
-

Change globally

- -

- - - - - - - -
- Unsubscribing from - Your other subscriptions -
- Turn on the confirmation checkbox and hit this button to - unsubscribe from this mailing list. Warning: - This action will be taken immediately! -

-

- You can view a list of all the other mailing lists at - for which you are a member. Use this if you want to - make the same membership option changes to this other - subscriptions. - -

-

-
- - - - - - -
- Your Password -
- -
-

Forgotten Your Password?

-
- Click this button to have your password emailed to your - membership address. -

-

- -
- 		- -
-

Change Your Password

- - - - - - - - -
New - password:
Again to - confirm:
- - -

Change globally. -
-
- -

- - -
- Your Subscription Options -
- -

-Current values are checked. - -

Note that some of the options have a Set globally -checkbox. Checking this field will cause the changes to be made to -every mailing list that you are a member of on . Click on -List my other subscriptions above to see which other mailing -lists you are subscribed to. -

- - - - - - - - - - - - - - - - - - - - - - - - - -
- - Mail delivery

- Set this option to Enabled to receive messages posted - to this mailing list. Set it to Disabled if you want - to stay subscribed, but don't want mail delivered to you for a - while (e.g. you're going on vacation). If you disable mail - delivery, don't forget to re-enable it when you come back; it - will not be automatically re-enabled. -

- Enabled
- Disabled

- Set globally -

- Set Digest Mode

- If you turn digest mode on, you'll get posts bundled together - (usually one per day but possibly more on busy lists), instead - of singly when they're sent. If digest mode is changed from - on to off, you may receive one last digest. -

- Off
- On -
- Get MIME or Plain Text Digests?

- Your mail reader may or may not support MIME digests. In - general MIME digests are preferred, but if you have a problem - reading them, select plain text digests. -

- MIME
- Plain Text

- Set globally -

- Receive your own posts to the list?

- Ordinarily, you will get a copy of every message you post to - the list. If you don't want to receive this copy, set this - option to No. -

- No
- Yes -
- Receive acknowledgement mail when you send mail to - the list?

-

- No
- Yes -
- Get password reminder email for this list?

- Once a month, you will get an email containing a password - reminder for every list at this host to which you are - subscribed. You can turn this off on a per-list basis by - selecting No for this option. If you turn off - password reminders for all the lists you are subscribed to, no - reminder email will be sent to you. -

- No
- Yes

- Set globally -

- Conceal yourself from subscriber list?

- When someone views the list membership, your email address is - normally shown (in an obscured fashion to thwart spam - harvesters). If you do not want your email address to show up - on this membership roster at all, select Yes for this option. -

- No
- Yes -
- What language do you prefer?

-

- -
- Which topic categories would you like to subscribe - to?

- By selecting one or more topics, you can filter the - traffic on the mailing list, so as to receive only a - subset of the messages. If a message matches one of - your selected topics, then you will get the message, - otherwise you will not. - -

If a message does not match any topic, the delivery - rule depends on the setting of the option below. If - you do not select any topics of interest, you will get - all the messages sent to the mailing list. -

- -
- Do you want to receive messages that do not match any - topic filter?

- - This option only takes effect if you've subscribed to - at least one topic above. It describes what the - default delivery rule is for messages that don't match - any topic filter. Selecting No says that if - the message does not match any topic filters, then you - won't get the message, while selecting Yes - says to deliver such non-matching messages to you. - -

If no topics of interest are selected above, then - you will receive every message sent to the mailing - list. -

- No
- Yes -
- Avoid duplicate copies of messages?

- - When you are listed explicitly in the To: or - Cc: headers of a list message, you can opt to - not receive another copy from the mailing list. - Select Yes to avoid receiving copies from the - mailing list; select No to receive copies. - -

If the list has member personalized messages - enabled, and you elect to receive copies, every copy - will have a X-Mailman-Copy: yes header added - to it. - -

- No
- Yes

- Set globally -

-
-
- -

- - - - - diff --git a/src/mailman/templates/en/postack.txt b/src/mailman/templates/en/postack.txt deleted file mode 100644 index 7f5836344..000000000 --- a/src/mailman/templates/en/postack.txt +++ /dev/null @@ -1,8 +0,0 @@ -Your message entitled - - $subject - -was successfully received by the $display_name mailing list. - -List info page: $listinfo_url -Your preferences: $optionsurl diff --git a/src/mailman/templates/en/postauth.txt b/src/mailman/templates/en/postauth.txt deleted file mode 100644 index 9e9ae8f57..000000000 --- a/src/mailman/templates/en/postauth.txt +++ /dev/null @@ -1,13 +0,0 @@ -As list administrator, your authorization is requested for the -following mailing list posting: - - List: $listname - From: $sender - Subject: $subject - -The message is being held because: - -$reasons - -At your convenience, visit your dashboard to approve or deny the -request. diff --git a/src/mailman/templates/en/postheld.txt b/src/mailman/templates/en/postheld.txt deleted file mode 100644 index ee769c8ae..000000000 --- a/src/mailman/templates/en/postheld.txt +++ /dev/null @@ -1,12 +0,0 @@ -Your mail to '$listname' with the subject - - $subject - -Is being held until the list moderator can review it for approval. - -The message is being held because: - -$reasons - -Either the message will get posted to the list, or you will receive -notification of the moderator's decision. diff --git a/src/mailman/templates/en/private.html b/src/mailman/templates/en/private.html deleted file mode 100644 index 28ac9bfc8..000000000 --- a/src/mailman/templates/en/private.html +++ /dev/null @@ -1,43 +0,0 @@ - - - %(realname)s Private Archives Authentication - - -

-%(message)s - - - - - - - - - - - - - - - -
- %(realname)s Private - Archives Authentication -
Email address:
Password:
-
-

Important: From this point on, you - must have cookies enabled in your browser, otherwise - you will have to re-authenticate with every operation. - -

Session cookies are used in Mailman's - private archive interface so that you don't need to - re-authenticate with every operation. This - cookie will expire automatically when you exit your browser, or - you can explicitly expire the cookie by visiting your - member options page and clicking the - Log out button. -

- - diff --git a/src/mailman/templates/en/probe.txt b/src/mailman/templates/en/probe.txt deleted file mode 100644 index 98eaa310d..000000000 --- a/src/mailman/templates/en/probe.txt +++ /dev/null @@ -1,20 +0,0 @@ -This is a probe message. You can ignore this message. - -The $listname mailing list has received a number of bounces from you, -indicating that there may be a problem delivering messages to $address. A -sample is attached below. Please examine this message to make sure there are -no problems with your email address. You may want to check with your mail -administrator for more help. - -You don't need to do anything to remain an enabled member of the mailing list. -You can also visit your membership page at - - $optionsurl - -On your membership page, you can change various delivery options such -as your email address and whether you get digests or not. - -If you have any questions or problems, you can contact the mailing list owner -at - - $owneraddr diff --git a/src/mailman/templates/en/refuse.txt b/src/mailman/templates/en/refuse.txt deleted file mode 100644 index ee8eba9b3..000000000 --- a/src/mailman/templates/en/refuse.txt +++ /dev/null @@ -1,13 +0,0 @@ -Your request to the $listname mailing list - - $request - -has been rejected by the list moderator. The moderator gave the -following reason for rejecting your request: - -"$reason" - -Any questions or comments should be directed to the list administrator -at: - - $adminaddr diff --git a/src/mailman/templates/en/roster.html b/src/mailman/templates/en/roster.html deleted file mode 100644 index aa5392ae4..000000000 --- a/src/mailman/templates/en/roster.html +++ /dev/null @@ -1,52 +0,0 @@ - - - <MM-List-Name> Subscribers - - - - -

- - - - - - - - - - - - - - - -
- - Subscribers -
- -

-

- -

Click on your address to visit your subscription - options page.
(Parenthesized entries have list delivery - disabled.)

-
-
- - Non-digested Members of : -
- 		-
- Digested - Members of : -
-
-

-

-

-

- - - diff --git a/src/mailman/templates/en/subauth.txt b/src/mailman/templates/en/subauth.txt deleted file mode 100644 index 041be5e55..000000000 --- a/src/mailman/templates/en/subauth.txt +++ /dev/null @@ -1,5 +0,0 @@ -Your authorization is required for a mailing list subscription request -approval: - - For: $username - List: $listname diff --git a/src/mailman/templates/en/subscribe.html b/src/mailman/templates/en/subscribe.html deleted file mode 100644 index 20373877b..000000000 --- a/src/mailman/templates/en/subscribe.html +++ /dev/null @@ -1,8 +0,0 @@ - -<MM-List-Name> Subscription results - -

Subscription results

- - - - diff --git a/src/mailman/templates/en/unrecognized.txt b/src/mailman/templates/en/unrecognized.txt deleted file mode 100644 index 16bd19e79..000000000 --- a/src/mailman/templates/en/unrecognized.txt +++ /dev/null @@ -1,7 +0,0 @@ -The attached message was received as a bounce, but either the bounce format -was not recognized, or no member addresses could be extracted from it. This -mailing list has been configured to send all unrecognized bounce messages to -the list administrator(s). - -For more information see: -$adminurl diff --git a/src/mailman/templates/en/unsub.txt b/src/mailman/templates/en/unsub.txt deleted file mode 100644 index b08f65bae..000000000 --- a/src/mailman/templates/en/unsub.txt +++ /dev/null @@ -1,23 +0,0 @@ -Mailing list removal confirmation notice for mailing list %(listname)s - -We have received a request%(remote)s for the removal of your email -address, "%(email)s" from the %(listaddr)s mailing list. To confirm -that you want to be removed from this mailing list, simply reply to -this message, keeping the Subject: header intact. Or visit this web -page: - - %(confirmurl)s - -Or include the following line -- and only the following line -- in a -message to %(requestaddr)s: - - confirm %(cookie)s - -Note that simply sending a `reply' to this message should work from -most mail readers, since that usually leaves the Subject: line in -the right form (additional "Re:" text in the Subject: is okay). - -If you do not wish to be removed from this list, please simply -disregard this message. If you think you are being maliciously -removed from the list, or have any other questions, send them to -%(listadmin)s. diff --git a/src/mailman/templates/en/unsubauth.txt b/src/mailman/templates/en/unsubauth.txt deleted file mode 100644 index e6a6a82f2..000000000 --- a/src/mailman/templates/en/unsubauth.txt +++ /dev/null @@ -1,11 +0,0 @@ -Your authorization is required for a mailing list unsubscription -request approval: - - By: $email - From: $listname - -At your convenience, visit: - - $admindb_url - -to process the request. diff --git a/src/mailman/templates/en/userpass.txt b/src/mailman/templates/en/userpass.txt deleted file mode 100644 index 2a53a846e..000000000 --- a/src/mailman/templates/en/userpass.txt +++ /dev/null @@ -1,24 +0,0 @@ -You, or someone posing as you, has requested a password reminder for -your membership on the mailing list %(fqdn_lname)s. You will need -this password in order to change your membership options (e.g. do you -want regular delivery or digest delivery), and having this password -makes it easier for you to unsubscribe from the mailing list. - -You are subscribed with the address: %(user)s - -Your %(listname)s password is: %(password)s - -To make changes to your membership options, log in and visit your -options web page: - - %(options_url)s - -You can also make such changes via email by sending a message to: - - %(requestaddr)s - -with the text "help" in the subject or body. The automatic reply will -contain more detailed instructions. - -Questions or comments? Please send them to the %(listname)s mailing -list administrator at %(owneraddr)s. diff --git a/src/mailman/templates/en/welcome.txt b/src/mailman/templates/en/welcome.txt deleted file mode 100644 index 503a422cd..000000000 --- a/src/mailman/templates/en/welcome.txt +++ /dev/null @@ -1,25 +0,0 @@ -Welcome to the "$list_name" mailing list! - -To post to this list, send your email to: - - $fqdn_listname - -General information about the mailing list is at: - - $listinfo_uri - -If you ever want to unsubscribe or change your options (eg, switch to or -from digest mode, change your password, etc.), visit your subscription -page at: - - $user_options_uri - -You can also make such adjustments via email by sending a message to: - - $list_requests - -with the word 'help' in the subject or body (don't include the quotes), and -you will get back a message with instructions. You will need your password to -change your options, but for security purposes, this email is not included -here. There is also a button on your options page that will send your current -password to you. diff --git a/src/mailman/testing/documentation.py b/src/mailman/testing/documentation.py index f3a6e335a..29629fe1b 100644 --- a/src/mailman/testing/documentation.py +++ b/src/mailman/testing/documentation.py @@ -25,7 +25,8 @@ from inspect import isfunction, ismethod from mailman import public from mailman.app.lifecycle import create_list from mailman.config import config -from mailman.testing.helpers import call_api, specialized_message_from_string +from mailman.testing.helpers import ( + call_api, get_queue_messages, specialized_message_from_string, subscribe) from mailman.testing.layers import SMTPLayer @@ -92,7 +93,7 @@ def call_http(url, data=None, method=None, username=None, password=None): content, response = call_api(url, data, method, username, password) if content is None: for header in sorted(response): - print('{0}: {1}'.format(header, response[header])) + print('{}: {}'.format(header, response[header])) return None return content @@ -145,9 +146,11 @@ def setup(testobj): testobj.globs['dump_json'] = dump_json testobj.globs['dump_msgdata'] = dump_msgdata testobj.globs['dump_list'] = dump_list + testobj.globs['get_queue_messages'] = get_queue_messages testobj.globs['message_from_string'] = specialized_message_from_string testobj.globs['smtpd'] = SMTPLayer.smtpd testobj.globs['stop'] = stop + testobj.globs['subscribe'] = subscribe testobj.globs['transaction'] = config.db # Add this so that cleanups can be automatically added by the doctest. testobj.globs['cleanups'] = [] diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index a87dd58ec..a57d883c7 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -21,7 +21,6 @@ import os import json import time import uuid -import errno import shutil import signal import socket @@ -228,11 +227,8 @@ def get_lmtp_client(quiet=False): if not quiet: print(response) return lmtp - except IOError as error: - if error.errno == errno.ECONNREFUSED: - time.sleep(0.1) - else: - raise + except ConnectionRefusedError: + time.sleep(0.1) else: raise RuntimeError('Connection refused') @@ -256,18 +252,16 @@ def get_nntp_server(cleanups): @public -def wait_for_webservice(): +def wait_for_webservice(hostname=None, port=None): """Wait for the REST server to start serving requests.""" + hostname = config.webservice.hostname if hostname is None else hostname + port = int(config.webservice.port) if port is None else port until = datetime.datetime.now() + as_timedelta(config.devmode.wait) while datetime.datetime.now() < until: try: - socket.socket().connect((config.webservice.hostname, - int(config.webservice.port))) - except IOError as error: - if error.errno == errno.ECONNREFUSED: - time.sleep(0.1) - else: - raise + socket.socket().connect((hostname, port)) + except ConnectionRefusedError: + time.sleep(0.1) else: break else: @@ -475,6 +469,9 @@ def reset_the_world(): for filename in filenames: os.remove(os.path.join(dirpath, filename)) shutil.rmtree(dirpath) + # Remove all the cache subdirectories, recursively. + for dirname in os.listdir(config.CACHE_DIR): + shutil.rmtree(os.path.join(config.CACHE_DIR, dirname)) # Reset the global style manager. getUtility(IStyleManager).populate() # Remove all dynamic header-match rules. diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index a282b2803..3157809e6 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -158,7 +158,7 @@ class ConfigLayer(MockAndMonkeyLayer): [logging.$name] propagate: yes level: debug - """), dict(name=sub_name, path=path)) + """), None, dict(name=sub_name, path=path)) # The root logger will already have a handler, but it's not the right # handler. Remove that and set our own. if cls.stderr: @@ -200,9 +200,7 @@ class ConfigLayer(MockAndMonkeyLayer): def testSetUp(cls): # Add an example domain. with transaction(): - getUtility(IDomainManager).add( - 'example.com', 'An example domain.', - 'http://lists.example.com') + getUtility(IDomainManager).add('example.com', 'An example domain.') @classmethod def testTearDown(cls): diff --git a/src/mailman/testing/mailman-fr.mo b/src/mailman/testing/mailman-fr.mo index 5758baf60..539d8760d 100644 Binary files a/src/mailman/testing/mailman-fr.mo and b/src/mailman/testing/mailman-fr.mo differ diff --git a/src/mailman/testing/mailman-fr.po b/src/mailman/testing/mailman-fr.po index ca4ce8c32..d7061f74e 100644 --- a/src/mailman/testing/mailman-fr.po +++ b/src/mailman/testing/mailman-fr.po @@ -20,9 +20,8 @@ msgid "" "Send $display_name mailing list submissions to\n" "\t$got_list_email\n" "\n" -"To subscribe or unsubscribe via the World Wide Web, visit\n" -"\t$got_listinfo_url\n" -"or, via email, send a message with subject or body 'help' to\n" +"To subscribe or unsubscribe via email, send a message with subject or body\n" +"'help' to\n" "\t$got_request_email\n" "\n" "You can reach the person managing the list at\n" @@ -34,11 +33,8 @@ msgstr "" "Envoyez vos messages pour la liste $display_name à\n" "\t$got_list_email\n" "\n" -"Pour vous (dés)abonner par le web, consultez\n" -"\t$got_listinfo_url\n" -"\n" -"ou, par courriel, envoyez un message avec « help » dans le corps\n" -"ou dans le sujet à\n" +"Pour vous (dés)abonner par courriel, envoyez un message avec « help » dans\n" +"le corps ou dans le sujet à\n" "\t$got_request_email\n" "\n" "Vous pouvez contacter l'administrateur de la liste à l'adresse\n" diff --git a/src/mailman/utilities/i18n.py b/src/mailman/utilities/i18n.py index e289a92be..1df8ede7e 100644 --- a/src/mailman/utilities/i18n.py +++ b/src/mailman/utilities/i18n.py @@ -24,9 +24,7 @@ from itertools import product from mailman import public from mailman.config import config from mailman.core.constants import system_preferences -from mailman.core.i18n import _ from mailman.interfaces.errors import MailmanError -from mailman.utilities.string import expand, wrap as wrap_text from pkg_resources import resource_filename @@ -159,46 +157,3 @@ def find(template_file, mlist=None, language=None, _trace=False): print(' FOUND:', path, file=sys.stderr) return path, fp raise TemplateNotFoundError(template_file) - - -@public -def make(template_file, mlist=None, language=None, wrap=True, - _trace=False, **kw): - """Locate and 'make' a template file. - - The template file is located as with `find()`, and the resulting text is - optionally wrapped and interpolated with the keyword argument dictionary. - - :param template_file: The name of the template file to search for. - :type template_file: string - :param mlist: Optional mailing list used as the context for - searching for the template file. The list's preferred language will - influence the search, as will the list's data directory. - :type mlist: `IMailingList` - :param language: Optional language code, which influences the search. - :type language: string - :param wrap: When True, wrap the text. - :type wrap: bool - :param _trace: Passed through to ``find()``, this enables printing of - debugging information during template search. - :type _trace: bool - :param **kw: Keyword arguments for template interpolation. - :return: The interpolated text. - :rtype: string - :raises TemplateNotFoundError: when the template could not be found. - """ - path, fp = find(template_file, mlist, language, _trace) - try: - # XXX Removing the trailing newline is a hack carried over from - # Mailman 2. The (stripped) template text is then passed through the - # translation catalog. This ensures that the translated text is - # unicode, and also allows for volunteers to translate the templates - # into the language catalogs. - template = _(fp.read()[:-1]) - finally: - fp.close() - assert isinstance(template, str), 'Translated template is not a string' - text = expand(template, kw) - if wrap: - return wrap_text(text) - return text diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index 1f350205f..c908a2ada 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -20,13 +20,12 @@ import os import re import sys -import codecs import logging import datetime from mailman import public from mailman.config import config -from mailman.handlers.decorate import decorate, decorate_template +from mailman.handlers.decorate import decorate_template from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.address import IEmailValidator from mailman.interfaces.archiver import ArchivePolicy @@ -41,11 +40,11 @@ from mailman.interfaces.mailinglist import ( SubscriptionPolicy) from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole from mailman.interfaces.nntp import NewsgroupModeration +from mailman.interfaces.template import ITemplateManager from mailman.interfaces.usermanager import IUserManager from mailman.utilities.filesystem import makedirs from mailman.utilities.i18n import search from sqlalchemy import Boolean -from urllib.error import URLError from zope.component import getUtility log = logging.getLogger('mailman.error') @@ -376,43 +375,27 @@ def import_config_pck(mlist, config_dict): # special `mailman:` scheme indicating a file system path. What we do # here is look to see if the list's decoration is different than the # default, and if so, we'll write the new decoration template to a - # `mailman:` scheme path. + # `mailman:` scheme path, then add the template to the template manager. convert_to_uri = { - 'welcome_msg': 'welcome_message_uri', - 'goodbye_msg': 'goodbye_message_uri', - 'msg_header': 'header_uri', - 'msg_footer': 'footer_uri', - 'digest_header': 'digest_header_uri', - 'digest_footer': 'digest_footer_uri', + 'welcome_msg': 'list:user:notice:welcome', + 'goodbye_msg': 'list:user:notice:goodbye', + 'msg_header': 'list:member:regular:header', + 'msg_footer': 'list:member:regular:footer', + 'digest_header': 'list:member:digest:header', + 'digest_footer': 'list:member:digest:footer', } # The best we can do is convert only the most common ones. These are # order dependent; the longer substitution with the common prefix must # show up earlier. convert_placeholders = [ - ('%(real_name)s@%(host_name)s', '$fqdn_listname'), + ('%(real_name)s@%(host_name)s', '$listname'), ('%(real_name)s', '$display_name'), - ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s', - '$listinfo_uri'), + # The generic footers no longer have URLs in them. + ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''), ] # Collect defaults. + manager = getUtility(ITemplateManager) defaults = {} - for oldvar, newvar in convert_to_uri.items(): - default_value = getattr(mlist, newvar, None) - if not default_value: - continue - # Check if the value changed from the default. - try: - default_text = decorate(mlist, default_value) - except (URLError, KeyError): - # Use case: importing the old a@ex.com into b@ex.com. We can't - # check if it changed from the default so don't import, we may do - # more harm than good and it's easy to change if needed. - # TESTME - print('Unable to convert mailing list attribute:', oldvar, - 'with old value "{}"'.format(default_value), - file=sys.stderr) - continue - defaults[newvar] = (default_value, default_text) for oldvar, newvar in convert_to_uri.items(): if oldvar not in config_dict: continue @@ -442,17 +425,18 @@ def import_config_pck(mlist, config_dict): expanded_text.strip() == default_text.strip()): # Keep the default. continue - # Write the custom value to the right file. + # Write the custom value to the right file and add it to the template + # manager for real. base_uri = 'mailman:///$listname/$language/' if default_value: filename = default_value.rpartition('/')[2] else: - filename = '{}.txt'.format(newvar[:-4]) + filename = '{}.txt'.format(newvar.replace(':', '_')) if not default_value or not default_value.startswith(base_uri): - setattr(mlist, newvar, base_uri + filename) + manager.set(newvar, mlist.list_id, base_uri + filename) filepath = list(search(filename, mlist))[0] makedirs(os.path.dirname(filepath)) - with codecs.open(filepath, 'w', encoding='utf-8') as fp: + with open(filepath, 'w', encoding='utf-8') as fp: fp.write(text) # Import rosters. regulars_set = set(config_dict.get('members', {})) diff --git a/src/mailman/utilities/protocols.py b/src/mailman/utilities/protocols.py new file mode 100644 index 000000000..5f447c465 --- /dev/null +++ b/src/mailman/utilities/protocols.py @@ -0,0 +1,101 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Various URL protocol support.""" + +import requests + +from mailman import public +from mailman.interfaces.languages import ILanguageManager +from mailman.interfaces.listmanager import IListManager +from mailman.utilities.i18n import TemplateNotFoundError, find +from urllib.error import URLError +from urllib.parse import urlparse +from zope.component import getUtility + +COMMASPACE = ', ' + + +@public +def get(url, **kws): + parsed = urlparse(url) + if parsed.scheme in ('http', 'https'): + response = requests.get(url, **kws) + response.raise_for_status() + return response.text + if parsed.scheme == 'file': + mode = kws.pop('mode', 'r') + arguments = dict(mode=mode) + if 'encoding' in kws or 'b' not in mode: + arguments['encoding'] = kws.pop('encoding', 'utf-8') + if len(kws) > 0: + raise ValueError('Unexpected arguments: {}'.format( + COMMASPACE.join(sorted(kws)))) + with open(parsed.path, **arguments) as fp: + return fp.read() + if parsed.scheme == 'mailman': + mlist = code = None + if len(kws) > 0: + raise ValueError('Unexpected arguments: {}'.format( + COMMASPACE.join(sorted(kws)))) + # The path can contain one, two, or three components. Since no empty + # path components are legal, filter them out. + parts = [p for p in parsed.path.split('/') if p] + if len(parts) == 0: + raise URLError('No template specified') + elif len(parts) == 1: + template = parts[0] + elif len(parts) == 2: + part0, template = parts + # Is part0 a language code or a mailing list? This is rather + # tricky because if it's a mailing list, it could be a list-id and + # that will contain dots, as could the language code. + language = getUtility(ILanguageManager).get(part0) + if language is None: + list_manager = getUtility(IListManager) + # part0 must be a fqdn-listname or list-id. + mlist = (list_manager.get(part0) + if '@' in part0 else + list_manager.get_by_list_id(part0)) + if mlist is None: + raise URLError('Bad language or list name') + else: + code = language.code + elif len(parts) == 3: + part0, code, template = parts + # part0 could be an fqdn-listname or a list-id. + mlist = (getUtility(IListManager).get(part0) + if '@' in part0 else + getUtility(IListManager).get_by_list_id(part0)) + if mlist is None: + raise URLError('Missing list') + language = getUtility(ILanguageManager).get(code) + if language is None: + raise URLError('No such language') + code = language.code + else: + raise URLError('No such file') + # Find the template, mutating any missing template exception. + try: + path, fp = find(template, mlist, code) + except TemplateNotFoundError: + raise URLError('No such file') + try: + return fp.read() + finally: + fp.close() + raise URLError(url) diff --git a/src/mailman/utilities/string.py b/src/mailman/utilities/string.py index 8394aa5e6..2694877ee 100644 --- a/src/mailman/utilities/string.py +++ b/src/mailman/utilities/string.py @@ -22,6 +22,7 @@ import logging from email.errors import HeaderParseError from email.header import decode_header, make_header from mailman import public +from mailman.config import config from string import Template, whitespace from textwrap import TextWrapper, dedent @@ -33,18 +34,40 @@ log = logging.getLogger('mailman.error') @public -def expand(template, substitutions, template_class=Template): +def expand(template, mlist=None, extras=None, template_class=Template): """Expand string template with substitutions. :param template: A PEP 292 $-string template. :type template: string - :param substitutions: The substitutions dictionary. - :type substitutions: dict + :param mlist: Optional mailing list. If given, the standard set of + list-specific substitution variables are used automatically. + :type mlist: `IMailingList` + :param extras: An additional substitutions dictionary. These are used to + augment any standard, list-specific substitutions. + :type extras: dict :param template_class: The template class to use. :type template_class: class :return: The substituted string. :rtype: string """ + substitutions = dict( + site_email=config.mailman.site_owner, + ) + if mlist is not None: + substitutions.update(dict( + listname=mlist.fqdn_listname, + list_id=mlist.list_id, + display_name=mlist.display_name, + short_listname=mlist.list_name, + domain=mlist.mail_host, + description=mlist.description, + info=mlist.info, + request_email=mlist.request_address, + owner_email=mlist.owner_address, + language=mlist.preferred_language.code, + )) + if extras is not None: + substitutions.update(extras) return template_class(template).safe_substitute(substitutions) @@ -119,10 +142,12 @@ def wrap(text, column=70, honor_leading_ws=True): wrapped_paragraphs = [] # The dedented wrapper. wrapper = TextWrapper(width=column, + break_on_hyphens=False, fix_sentence_endings=True) # The indented wrapper. For this one, we'll clobber initial_indent and # subsequent_indent as needed per indented chunk of text. iwrapper = TextWrapper(width=column, + break_on_hyphens=False, fix_sentence_endings=True, ) add_paragraph_break = False diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py index b2d48891b..d2729e8e1 100644 --- a/src/mailman/utilities/tests/test_import.py +++ b/src/mailman/utilities/tests/test_import.py @@ -31,22 +31,23 @@ from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bans import IBanManager from mailman.interfaces.bounce import UnrecognizedBounceDisposition +from mailman.interfaces.domain import IDomainManager from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.mailinglist import ( IAcceptableAliasSet, SubscriptionPolicy) from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.interfaces.nntp import NewsgroupModeration -from mailman.interfaces.templates import ITemplateLoader +from mailman.interfaces.template import ITemplateLoader, ITemplateManager from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import LogFileMark from mailman.testing.layers import ConfigLayer from mailman.utilities.filesystem import makedirs from mailman.utilities.importer import ( Import21Error, check_language_code, import_config_pck) -from mailman.utilities.string import expand from pickle import load from pkg_resources import resource_filename from unittest import mock +from urllib.error import URLError from zope.component import getUtility @@ -638,16 +639,17 @@ class TestConvertToURI(unittest.TestCase): # -> %(listinfo_uri)s layer = ConfigLayer + maxDiff = None def setUp(self): self._mlist = create_list('blank@example.com') self._conf_mapping = dict( - welcome_msg='welcome_message_uri', - goodbye_msg='goodbye_message_uri', - msg_header='header_uri', - msg_footer='footer_uri', - digest_header='digest_header_uri', - digest_footer='digest_footer_uri', + welcome_msg='list:user:notice:welcome', + goodbye_msg='list:user:notice:goodbye', + msg_header='list:member:regular:header', + msg_footer='list:member:regular:footer', + digest_header='list:member:digest:header', + digest_footer='list:member:digest:footer', ) self._pckdict = dict() @@ -655,8 +657,7 @@ class TestConvertToURI(unittest.TestCase): for oldvar, newvar in self._conf_mapping.items(): self._pckdict[str(oldvar)] = b'TEST VALUE' import_config_pck(self._mlist, self._pckdict) - newattr = getattr(self._mlist, newvar) - text = decorate(self._mlist, newattr) + text = decorate(newvar, self._mlist) self.assertEqual( text, 'TEST VALUE', 'Old variable %s was not properly imported to %s' @@ -664,21 +665,13 @@ class TestConvertToURI(unittest.TestCase): def test_substitutions(self): test_text = ('UNIT TESTING %(real_name)s mailing list\n' - '%(real_name)s@%(host_name)s\n' - '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s') + '%(real_name)s@%(host_name)s') expected_text = ('UNIT TESTING $display_name mailing list\n' - '$fqdn_listname\n' - '$listinfo_uri') + '$listname') for oldvar, newvar in self._conf_mapping.items(): self._pckdict[str(oldvar)] = str(test_text) import_config_pck(self._mlist, self._pckdict) - newattr = getattr(self._mlist, newvar) - template_uri = expand(newattr, dict( - listname=self._mlist.fqdn_listname, - language=self._mlist.preferred_language.code, - )) - loader = getUtility(ITemplateLoader) - text = loader.get(template_uri) + text = getUtility(ITemplateLoader).get(newvar, self._mlist) self.assertEqual( text, expected_text, 'Old variables were not converted for %s' % newvar) @@ -689,35 +682,53 @@ class TestConvertToURI(unittest.TestCase): '_______________________________________________\n' '%(real_name)s mailing list\n' '%(real_name)s@%(host_name)s\n' - '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s' + '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n' ) + loader = getUtility(ITemplateLoader) for oldvar in ('msg_footer', 'digest_footer'): newvar = self._conf_mapping[oldvar] self._pckdict[str(oldvar)] = str(default_msg_footer) - old_value = getattr(self._mlist, newvar) + try: + old_value = loader.get(newvar, self._mlist) + except URLError: + old_value = None import_config_pck(self._mlist, self._pckdict) - new_value = getattr(self._mlist, newvar) + try: + new_value = loader.get(newvar, self._mlist) + except URLError: + new_value = None self.assertEqual( old_value, new_value, - 'Default value was not preserved for %s' % newvar) + '{} changed unexpectedly: {} != {}'.format( + newvar, old_value, new_value)) def test_keep_default_if_fqdn_changed(self): # Use case: importing the old a@ex.com into b@ex.com. We can't check # if it changed from the default so don't import. We may do more harm # than good and it's easy to change if needed. test_value = b'TEST-VALUE' + # We need an IDomain for this mail_host. + getUtility(IDomainManager).add('test.example.com') + manager = getUtility(ITemplateManager) for oldvar, newvar in self._conf_mapping.items(): self._mlist.mail_host = 'example.com' self._pckdict['mail_host'] = b'test.example.com' self._pckdict[str(oldvar)] = test_value - old_value = getattr(self._mlist, newvar) + try: + old_value = manager.get(newvar, 'blank.example.com') + except URLError: + old_value = None # Suppress warning messages in the test output. with mock.patch('sys.stderr'): import_config_pck(self._mlist, self._pckdict) - new_value = getattr(self._mlist, newvar) + try: + new_value = manager.get(newvar, 'test.example.com') + except URLError: + new_value = None self.assertEqual( old_value, new_value, - 'Default value was not preserved for %s' % newvar) + '{} changed unexpectedly: {} != {}'.format( + newvar, old_value, new_value)) def test_unicode(self): # non-ascii templates @@ -725,10 +736,11 @@ class TestConvertToURI(unittest.TestCase): self._pckdict[str(oldvar)] = b'Ol\xe1!' import_config_pck(self._mlist, self._pckdict) for oldvar, newvar in self._conf_mapping.items(): - newattr = getattr(self._mlist, newvar) - text = decorate(self._mlist, newattr) + text = decorate(newvar, self._mlist) expected = u'Ol\ufffd!' - self.assertEqual(text, expected) + self.assertEqual( + text, expected, + '{} -> {} did not get converted'.format(oldvar, newvar)) def test_unicode_in_default(self): # What if the default template is already in UTF-8? For example, if @@ -736,13 +748,13 @@ class TestConvertToURI(unittest.TestCase): footer = b'\xe4\xb8\xad $listinfo_uri' footer_path = os.path.join( config.VAR_DIR, 'templates', 'lists', - 'blank@example.com', 'en', 'footer-generic.txt') + 'blank@example.com', 'en', 'footer.txt') makedirs(os.path.dirname(footer_path)) with open(footer_path, 'wb') as fp: fp.write(footer) self._pckdict['msg_footer'] = b'NEW-VALUE' import_config_pck(self._mlist, self._pckdict) - text = decorate(self._mlist, self._mlist.footer_uri) + text = decorate('list:member:regular:footer', self._mlist) self.assertEqual(text, 'NEW-VALUE') diff --git a/src/mailman/utilities/tests/test_protocols.py b/src/mailman/utilities/tests/test_protocols.py new file mode 100644 index 000000000..dc5e46e6a --- /dev/null +++ b/src/mailman/utilities/tests/test_protocols.py @@ -0,0 +1,174 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the protocol support. + +For convenience, we currently don't test the http: and https: schemes here. +These are tested fairly well in the template cache tests. We probably +eventually want to refactor that for test isolation. +""" + + +import os +import unittest + +from contextlib import ExitStack +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.testing.layers import ConfigLayer +from mailman.utilities import protocols +from tempfile import TemporaryDirectory +from urllib.error import URLError + + +class TestProtocols(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + resources = ExitStack() + self.addCleanup(resources.close) + self.var_dir = resources.enter_context(TemporaryDirectory()) + config.push('template config', """\ + [paths.testing] + var_dir: {} + """.format(self.var_dir)) + resources.callback(config.pop, 'template config') + # Put a demo template in the site directory. + path = os.path.join(self.var_dir, 'templates', 'site', 'en') + os.makedirs(path) + with open(os.path.join(path, 'demo.txt'), 'w') as fp: + print('Test content', end='', file=fp) + self._mlist = create_list('test@example.com') + + def test_file(self): + with TemporaryDirectory() as tempdir: + path = os.path.join(tempdir, 'my-file') + with open(path, 'w', encoding='utf-8') as fp: + print('Some contents', end='', file=fp) + contents = protocols.get('file:///{}'.format(path)) + self.assertEqual(contents, 'Some contents') + + def test_file_binary(self): + with TemporaryDirectory() as tempdir: + path = os.path.join(tempdir, 'my-file') + with open(path, 'wb') as fp: + fp.write(b'xxx') + contents = protocols.get('file:///{}'.format(path), mode='rb') + self.assertEqual(contents, b'xxx') + + def test_file_ascii(self): + with TemporaryDirectory() as tempdir: + path = os.path.join(tempdir, 'my-file') + with open(path, 'w', encoding='us-ascii') as fp: + print('Some contents', end='', file=fp) + contents = protocols.get('file:///{}'.format(path), + encoding='us-ascii') + self.assertEqual(contents, 'Some contents') + + def test_mailman_internal_uris(self): + # mailman://demo.txt + content = protocols.get('mailman:///demo.txt') + self.assertEqual(content, 'Test content') + + def test_mailman_internal_uris_twice(self): + # mailman:///demo.txt + content = protocols.get('mailman:///demo.txt') + self.assertEqual(content, 'Test content') + content = protocols.get('mailman:///demo.txt') + self.assertEqual(content, 'Test content') + + def test_mailman_uri_with_language(self): + content = protocols.get('mailman:///en/demo.txt') + self.assertEqual(content, 'Test content') + + def test_mailman_uri_with_english_fallback(self): + content = protocols.get('mailman:///it/demo.txt') + self.assertEqual(content, 'Test content') + + def test_mailman_uri_with_list_name(self): + content = protocols.get('mailman:///test@example.com/demo.txt') + self.assertEqual(content, 'Test content') + + def test_mailman_full_uri(self): + content = protocols.get('mailman:///test@example.com/en/demo.txt') + self.assertEqual(content, 'Test content') + + def test_mailman_full_uri_with_english_fallback(self): + content = protocols.get('mailman:///test@example.com/it/demo.txt') + self.assertEqual(content, 'Test content') + + def test_uri_not_found(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///missing.txt') + self.assertEqual(cm.exception.reason, 'No such file') + + def test_shorter_url_error(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///') + self.assertEqual(cm.exception.reason, 'No template specified') + + def test_short_url_error(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman://') + self.assertEqual(cm.exception.reason, 'No template specified') + + def test_bad_language(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///xx/demo.txt') + self.assertEqual(cm.exception.reason, 'Bad language or list name') + + def test_bad_mailing_list(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///missing@example.com/demo.txt') + self.assertEqual(cm.exception.reason, 'Bad language or list name') + + def test_missing_mailing_list(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///missing@example.com/it/demo.txt') + self.assertEqual(cm.exception.reason, 'Missing list') + + def test_no_such_language(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///test@example.com/xx/demo.txt') + self.assertEqual(cm.exception.reason, 'No such language') + + def test_too_many_path_components(self): + with self.assertRaises(URLError) as cm: + protocols.get('mailman:///missing@example.com/en/foo/demo.txt') + self.assertEqual(cm.exception.reason, 'No such file') + + def test_non_ascii(self): + # mailman://demo.txt with non-ascii content. + test_text = b'\xe4\xb8\xad' + path = os.path.join(self.var_dir, 'templates', 'site', 'it') + os.makedirs(path) + with open(os.path.join(path, 'demo.txt'), 'wb') as fp: + fp.write(test_text) + content = protocols.get('mailman:///it/demo.txt') + self.assertIsInstance(content, str) + self.assertEqual(content, test_text.decode('utf-8')) + + def test_bad_file_keyword(self): + self.assertRaises(ValueError, protocols.get, 'file:///etc/passwd', + invalid_keyword='yes') + + def test_bad_mailman_keyword(self): + self.assertRaises(ValueError, protocols.get, 'mailman:///demo.text', + invalid_keyword='yes') + + def test_bad_protocol(self): + self.assertRaises(URLError, protocols.get, 'unknown:///demo.text') diff --git a/src/mailman/utilities/tests/test_templates.py b/src/mailman/utilities/tests/test_templates.py index fcb0ecc1c..4e05f13c6 100644 --- a/src/mailman/utilities/tests/test_templates.py +++ b/src/mailman/utilities/tests/test_templates.py @@ -26,7 +26,7 @@ from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.languages import ILanguageManager from mailman.testing.layers import ConfigLayer -from mailman.utilities.i18n import TemplateNotFoundError, find, make, search +from mailman.utilities.i18n import TemplateNotFoundError, find, search from pkg_resources import resource_filename from zope.component import getUtility @@ -219,65 +219,3 @@ class TestFind(unittest.TestCase): with self.assertRaises(TemplateNotFoundError) as cm: find('missing.txt', self.mlist) self.assertEqual(cm.exception.template_file, 'missing.txt') - - -class TestMake(unittest.TestCase): - """Test template interpolation.""" - - layer = ConfigLayer - - def setUp(self): - self.var_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.var_dir) - config.push('template config', """\ - [paths.testing] - var_dir: {} - """.format(self.var_dir)) - self.addCleanup(config.pop, 'template config') - # The following MUST happen AFTER the push() above since pushing a new - # config also clears out the language manager. - getUtility(ILanguageManager).add('xx', 'utf-8', 'Xlandia') - self.mlist = create_list('test@example.com') - self.mlist.preferred_language = 'xx' - # Populate the template directories with a few fake templates. - path = os.path.join(self.var_dir, 'templates', 'site', 'xx') - os.makedirs(path) - with open(os.path.join(path, 'nosub.txt'), 'w') as fp: - print("""\ -This is a global template. -It has no substitutions. -It will be wrapped. -""", file=fp) - with open(os.path.join(path, 'subs.txt'), 'w') as fp: - print("""\ -This is a $kind template. -It has $howmany substitutions. -It will be wrapped. -""", file=fp) - with open(os.path.join(path, 'nowrap.txt'), 'w') as fp: - print("""\ -This is a $kind template. -It has $howmany substitutions. -It will not be wrapped. -""", file=fp) - - def test_no_substitutions(self): - self.assertEqual(make('nosub.txt', self.mlist), """\ -This is a global template. It has no substitutions. It will be -wrapped.""") - - def test_substitutions(self): - self.assertEqual(make('subs.txt', self.mlist, - kind='very nice', - howmany='a few'), """\ -This is a very nice template. It has a few substitutions. It will be -wrapped.""") - - def test_substitutions_no_wrap(self): - self.assertEqual(make('nowrap.txt', self.mlist, wrap=False, - kind='very nice', - howmany='a few'), """\ -This is a very nice template. -It has a few substitutions. -It will not be wrapped. -""") -- cgit v1.3.1