diff options
| -rw-r--r-- | src/mailman/bin/master.py | 4 | ||||
| -rw-r--r-- | src/mailman/commands/cli_control.py | 6 | ||||
| -rw-r--r-- | src/mailman/commands/cli_info.py | 17 | ||||
| -rw-r--r-- | src/mailman/commands/docs/control.txt | 2 | ||||
| -rw-r--r-- | src/mailman/commands/docs/info.txt | 43 | ||||
| -rw-r--r-- | src/mailman/commands/docs/unshunt.txt | 2 | ||||
| -rw-r--r-- | src/mailman/config/config.py | 102 | ||||
| -rw-r--r-- | src/mailman/config/mailman.cfg | 23 | ||||
| -rw-r--r-- | src/mailman/config/schema.cfg | 60 | ||||
| -rw-r--r-- | src/mailman/core/initialize.py | 1 | ||||
| -rw-r--r-- | src/mailman/queue/__init__.py | 5 | ||||
| -rw-r--r-- | src/mailman/testing/layers.py | 7 |
12 files changed, 233 insertions, 39 deletions
diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index cd29fb2df..721cc0091 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -507,7 +507,7 @@ def main(): # because Lock's constructor doesn't support a timeout. lock = acquire_lock(options.options.force) try: - with open(config.PIDFILE, 'w') as fp: + with open(config.PID_FILE, 'w') as fp: print >> fp, os.getpid() loop = Loop(lock, options.options.restartable, options.options.config) loop.install_signal_handlers() @@ -516,6 +516,6 @@ def main(): loop.loop() finally: loop.cleanup() - os.remove(config.PIDFILE) + os.remove(config.PID_FILE) finally: lock.unlock() diff --git a/src/mailman/commands/cli_control.py b/src/mailman/commands/cli_control.py index bd108ecd3..e6c9c83b3 100644 --- a/src/mailman/commands/cli_control.py +++ b/src/mailman/commands/cli_control.py @@ -127,11 +127,11 @@ class Start: def kill_watcher(sig): try: - with open(config.PIDFILE) as fp: + with open(config.PID_FILE) as fp: pid = int(fp.read().strip()) except (IOError, ValueError) as error: # For i18n convenience - print >> sys.stderr, _('PID unreadable in: $config.PIDFILE') + print >> sys.stderr, _('PID unreadable in: $config.PID_FILE') print >> sys.stderr, error print >> sys.stderr, _('Is the master even running?') return @@ -143,7 +143,7 @@ def kill_watcher(sig): print >> sys.stderr, _('No child with pid: $pid') print >> sys.stderr, error print >> sys.stderr, _('Stale pid file removed.') - os.unlink(config.PIDFILE) + os.unlink(config.PID_FILE) diff --git a/src/mailman/commands/cli_info.py b/src/mailman/commands/cli_info.py index b3ca02718..98a025673 100644 --- a/src/mailman/commands/cli_info.py +++ b/src/mailman/commands/cli_info.py @@ -50,6 +50,11 @@ class Info: action='store', help=_("""\ File to send the output to. If not given, standard output is used.""")) + command_parser.add_argument( + '-p', '--paths', + action='store_true', help=_("""\ + A more verbose output including the file system paths that Mailman + is using.""")) def process(self, args): """See `ICLISubCommand`.""" @@ -59,8 +64,18 @@ class Info: # We don't need to close output because that will happen # automatically when the script exits. output = open(args.output, 'w') - print >> output, MAILMAN_VERSION_FULL print >> output, 'Python', sys.version print >> output, 'config file:', config.filename print >> output, 'db url:', config.db.url + if args.paths: + print >> output, 'File system paths:' + longest = 0 + paths = {} + for attribute in dir(config): + if attribute.endswith('_DIR') or attribute.endswith('_FILE'): + paths[attribute] = getattr(config, attribute) + longest = max(longest, len(attribute)) + for attribute in sorted(paths): + print ' {0:{2}} = {1}'.format(attribute, paths[attribute], + longest) diff --git a/src/mailman/commands/docs/control.txt b/src/mailman/commands/docs/control.txt index 7bfaf9265..78a15b7b2 100644 --- a/src/mailman/commands/docs/control.txt +++ b/src/mailman/commands/docs/control.txt @@ -72,7 +72,7 @@ watcher process in the background. ... while datetime.now() < until: ... time.sleep(0.1) ... try: - ... with open(config.PIDFILE) as fp: + ... with open(config.PID_FILE) as fp: ... pid = int(fp.read().strip()) ... os.kill(pid, 0) ... except IOError as error: diff --git a/src/mailman/commands/docs/info.txt b/src/mailman/commands/docs/info.txt index f524d8f0a..53a1d531d 100644 --- a/src/mailman/commands/docs/info.txt +++ b/src/mailman/commands/docs/info.txt @@ -10,6 +10,7 @@ script 'mailman info'. By default, the info is printed to standard output. >>> class FakeArgs: ... output = None + ... paths = None >>> args = FakeArgs() >>> command.process(args) @@ -33,3 +34,45 @@ By passing in the -o/--output option, you can print the info to a file. ... config file: .../test.cfg db url: sqlite:.../mailman.db + +You can also get more verbose information, which contains a list of the file +system paths that Mailman is using. + + >>> args.output = None + >>> args.paths = True + >>> config.create_paths = False + >>> config.push('fhs', """ + ... [mailman] + ... layout: fhs + ... """) + >>> config.create_paths = True + +The File System Hierarchy layout is the same every by definition. + + >>> command.process(args) + GNU Mailman 3... + Python ... + ... + File system paths: + BIN_DIR = /sbin + CREATOR_PW_FILE = /var/lib/mailman/data/creator.pw + DATA_DIR = /var/lib/mailman/data + ETC_DIR = /etc + EXT_DIR = /etc/mailman.d + LIST_DATA_DIR = /var/lib/mailman/lists + LOCK_DIR = /var/lock/mailman + LOCK_FILE = /var/lock/mailman/master-qrunner.lck + LOG_DIR = /var/log/mailman + MESSAGES_DIR = /var/lib/mailman/messages + PID_FILE = /var/run/mailman/master-qrunner.pid + PRIVATE_ARCHIVE_FILE_DIR = /var/lib/mailman/archives/private + PUBLIC_ARCHIVE_FILE_DIR = /var/lib/mailman/archives/public + QUEUE_DIR = /var/spool/mailman + SITE_PW_FILE = /var/lib/mailman/data/adm.pw + VAR_DIR = /var/lib/mailman + + +Clean up +======== + + >>> config.pop('fhs') diff --git a/src/mailman/commands/docs/unshunt.txt b/src/mailman/commands/docs/unshunt.txt index 2c2ed5712..dcf71f3d1 100644 --- a/src/mailman/commands/docs/unshunt.txt +++ b/src/mailman/commands/docs/unshunt.txt @@ -23,6 +23,8 @@ Let's say there is a message in the shunt queue. ... """) >>> shuntq = config.switchboards['shunt'] + >>> len(list(shuntq.files)) + 0 >>> base_name = shuntq.enqueue(msg, {}) >>> len(list(shuntq.files)) 1 diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index ee27ff19d..37effc77c 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -32,6 +32,7 @@ import logging from lazr.config import ConfigSchema, as_boolean from pkg_resources import resource_stream +from string import Template from zope.component import getUtility from zope.interface import Interface, implements @@ -63,6 +64,9 @@ class Configuration: self.QFILE_SCHEMA_VERSION = version.QFILE_SCHEMA_VERSION self._config = None self.filename = None + # Whether to create run-time paths or not. This is for the test + # suite, which will set this to False until the test layer is set up. + self.create_paths = True # Create various registries. self.chains = {} self.rules = {} @@ -115,29 +119,8 @@ class Configuration: def _post_process(self): """Perform post-processing after loading the configuration files.""" - # Set up directories. - self.BIN_DIR = os.path.abspath(os.path.dirname(sys.argv[0])) - self.VAR_DIR = var_dir = self._config.mailman.var_dir - # Now that we've loaded all the configuration files we're going to - # load, set up some useful directories. - join = os.path.join - self.LIST_DATA_DIR = join(var_dir, 'lists') - self.LOG_DIR = join(var_dir, 'logs') - self.LOCK_DIR = lockdir = join(var_dir, 'locks') - self.DATA_DIR = datadir = join(var_dir, 'data') - self.ETC_DIR = etcdir = join(var_dir, 'etc') - self.SPAM_DIR = join(var_dir, 'spam') - self.EXT_DIR = join(var_dir, 'ext') - self.QUEUE_DIR = join(var_dir, 'qfiles') - self.MESSAGES_DIR = join(var_dir, 'messages') - self.PUBLIC_ARCHIVE_FILE_DIR = join(var_dir, 'archives', 'public') - self.PRIVATE_ARCHIVE_FILE_DIR = join(var_dir, 'archives', 'private') - # Other useful files - self.PIDFILE = join(datadir, 'master-qrunner.pid') - self.SITE_PW_FILE = join(datadir, 'adm.pw') - self.LISTCREATOR_PW_FILE = join(datadir, 'creator.pw') - self.CONFIG_FILE = join(etcdir, 'mailman.cfg') - self.LOCK_FILE = join(lockdir, 'master-qrunner') + # Expand and set up all directories. + self._expand_paths() # Set up the switchboards. from mailman.queue import Switchboard Switchboard.initialize() @@ -159,6 +142,74 @@ class Configuration: from mailman.core.i18n import _ _.default = self.mailman.default_language + def _expand_paths(self): + """Expand all configuration paths.""" + # Set up directories. + bin_dir = os.path.abspath(os.path.dirname(sys.argv[0])) + # Now that we've loaded all the configuration files we're going to + # load, set up some useful directories based on the settings in the + # configuration file. + layout = 'paths.' + self._config.mailman.layout + for category in self._config.getByCategory('paths'): + if category.name == layout: + break + else: + print >> sys.stderr, 'No path configuration found:', layout + sys.exit(1) + # First, collect all variables in a substitution dictionary. + substitutions = dict( + argv = bin_dir, + # Directories. + bin_dir = category.bin_dir, + data_dir = category.data_dir, + etc_dir = category.etc_dir, + ext_dir = category.ext_dir, + list_data_dir = category.list_data_dir, + lock_dir = category.lock_dir, + log_dir = category.log_dir, + messages_dir = category.messages_dir, + pipermail_private_dir = category.pipermail_private_dir, + pipermail_public_dir = category.pipermail_public_dir, + queue_dir = category.queue_dir, + var_dir = category.var_dir, + # Files. + creator_pw_file = category.creator_pw_file, + lock_file = category.lock_file, + pid_file = category.pid_file, + site_pw_file = category.site_pw_file, + ) + # Now, perform substitutions recursively until there are no more + # variables with $-vars in them, or until substitutions are not + # helping any more. + last_dollar_count = 0 + while True: + # Mutate the dictionary during iteration. + dollar_count = 0 + for key in substitutions.keys(): + raw_value = substitutions[key] + value = Template(raw_value).safe_substitute(substitutions) + if '$' in value: + # Still more work to do. + dollar_count += 1 + substitutions[key] = value + if dollar_count == 0: + break + if dollar_count == last_dollar_count: + print >> sys.stderr, 'Path expansion infloop detected' + sys.exit(1) + last_dollar_count = dollar_count + # Ensure that all paths are normalized and made absolute. Handle the + # few special cases first. Most of these are due to backward + # compatibility. + self.PUBLIC_ARCHIVE_FILE_DIR = substitutions.pop( + 'pipermail_public_dir') + self.PRIVATE_ARCHIVE_FILE_DIR = substitutions.pop( + 'pipermail_private_dir') + self.PID_FILE = substitutions.pop('pid_file') + for key in substitutions: + attribute = key.upper() + setattr(self, attribute, os.path.abspath(substitutions[key])) + @property def logger_configs(self): """Return all log config sections.""" @@ -173,8 +224,9 @@ class Configuration: def ensure_directories_exist(self): """Create all path directories if they do not exist.""" - for variable, directory in self.paths.items(): - makedirs(directory) + if self.create_paths: + for variable, directory in self.paths.items(): + makedirs(directory) @property def qrunner_configs(self): diff --git a/src/mailman/config/mailman.cfg b/src/mailman/config/mailman.cfg index a8a79ed30..075cc8038 100644 --- a/src/mailman/config/mailman.cfg +++ b/src/mailman/config/mailman.cfg @@ -18,6 +18,29 @@ # This is the absolute bare minimum base configuration file. User supplied # configurations are pushed onto this. +[paths.local] +# Directories as specified in schema.cfg, putting most stuff in +# /var/tmp/mailman + +[paths.dev] +# Convenient development layout where everything is put in the current +# directory. +var_dir: . + +[paths.fhs] +# Filesystem Hiearchy Standard 2.3 +# http://www.pathname.com/fhs/pub/fhs-2.3.html +bin_dir: /sbin +var_dir: /var/lib/mailman +queue_dir: /var/spool/mailman +log_dir: /var/log/mailman +lock_dir: /var/lock/mailman +etc_dir: /etc +ext_dir: /etc/mailman.d +pid_file: /var/run/mailman/master-qrunner.pid +creator_pw_file: $data_dir/creator.pw +site_pw_file: $data_dir/adm.pw + [language.en] [qrunner.archive] diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index eadfc5654..c72218bd5 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -34,9 +34,6 @@ site_owner: changeme@example.com # process. noreply_address: noreply -# Where all the runtime data will be kept. This directory must exist. -var_dir: /tmp/mailman - # The default language for this server. default_language: en @@ -62,6 +59,63 @@ pre_hook: # This runs after adapters are initialized. post_hook: +# Which paths.* file system layout to use. +layout: dev + + +[paths.master] +# Important directories for Mailman operation. These are defined here so that +# different layouts can be supported. For example, a developer layout would +# be different from a FHS layout. Most paths are based off the var_dir, and +# often just setting that will do the right thing for all the other paths. +# You might also have to set spool_dir though. +# +# Substitutions are allowed, but must be of the form $var where 'var' names a +# configuration variable in the paths.* section. Substitutions are expanded +# recursively until no more $-variables are present. Beware of infinite +# expansion loops! +# +# This is the root of the directory structure that Mailman will use to store +# its run-time data. +var_dir: /var/tmp/mailman +# This is where the Mailman queue files directories will be created. +queue_dir: $var_dir/queue +# This is the directory containing the Mailman 'qrunner' and 'master' commands +# if set to the string '$argv', it will be taken as the directory containing +# the 'bin/mailman' command. +bin_dir: $argv +# All list-specific data. +list_data_dir: $var_dir/lists +# Directory where log files go. +log_dir: $var_dir/logs +# Directory for system-wide locks. +lock_dir: $var_dir/locks +# Directory for system-wide data. +data_dir: $var_dir/data +# Directory for configuration files and such. +etc_dir: $var_dir/etc +# Directory containing Mailman plugins. +ext_dir: $var_dir/ext +# Directory where the default IMessageStore puts its messages. +messages_dir: $var_dir/messages +# Directory for public Pipermail archiver artifacts. +pipermail_public_dir: $var_dir/archives/public +# Directory for private Pipermail archiver artifacts. +pipermail_private_dir: $var_dir/archives/private +# +# There are also a number of paths to specific file locations that can be +# defined. For these, the directory containing the file must already exist, +# or be one of the directories created by Mailman as per above. +# +# This is where PID file for the master queue runner is stored. +pid_file: $var_dir/master-qrunner.pid +# The site administrators password [obsolete]. +site_pw_file: $var_dir/adm.pw +# The site list creator's password [obsolete]. +creator_pw_file: $var_dir/creator.pw +# Lock file. +lock_file: $lock_dir/master-qrunner.lck + [devmode] # Setting enabled to true enables certain safeguards and other behavior diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index e913af9c6..7c03a61db 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -120,7 +120,6 @@ def initialize_1(config_path=None, propagate_logs=None): config_path = None mailman.config.config.load(config_path) # Create the queue and log directories if they don't already exist. - mailman.config.config.ensure_directories_exist() mailman.core.logging.initialize(propagate_logs) diff --git a/src/mailman/queue/__init__.py b/src/mailman/queue/__init__.py index 829360330..4e8ba35e2 100644 --- a/src/mailman/queue/__init__.py +++ b/src/mailman/queue/__init__.py @@ -114,8 +114,9 @@ class Switchboard: 'Not a power of 2: {0}'.format(numslices)) self.name = name self.queue_directory = queue_directory - # Create the directory if it doesn't yet exist. - makedirs(self.queue_directory, 0770) + # If configured to, create the directory if it doesn't yet exist. + if config.create_paths: + makedirs(self.queue_directory, 0770) # Fast track for no slices self._lower = None self._upper = None diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 394cb9012..e7907b172 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -83,7 +83,9 @@ class ConfigLayer(MockAndMonkeyLayer): @classmethod def setUp(cls): - # Set up the basic configuration stuff. + # Set up the basic configuration stuff. Turn off path creation until + # we've pushed the testing config. + config.create_paths = False initialize.initialize_1(INHIBIT_CONFIG_FILE) assert cls.var_dir is None, 'Layer already set up' # Calculate a temporary VAR_DIR directory so that run-time artifacts @@ -100,10 +102,13 @@ class ConfigLayer(MockAndMonkeyLayer): # also write it out to a temp file for -C. test_config = dedent(""" [mailman] + layout: testing + [paths.testing] var_dir: %s """ % cls.var_dir) # Read the testing config and push it. test_config += resource_string('mailman.testing', 'testing.cfg') + config.create_paths = True config.push('test config', test_config) # Initialize everything else. initialize.initialize_2() |