Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 452806c

Browse files
vsajipvsajip
committed
Merged logging cookbook update from 3.2.
2 parents df97cbe + 0292fa9 commit 452806c

1 file changed

Lines changed: 228 additions & 0 deletions

File tree

‎Doc/howto/logging-cookbook.rst‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1343,3 +1343,231 @@ These are not "true" .gz files, as they are bare compressed data, with no
13431343
"container" such as you’d find in an actual gzip file. This snippet is just
13441344
for illustration purposes.
13451345

1346+
A more elaborate multiprocessing example
1347+
----------------------------------------
1348+
1349+
The following working example shows how logging can be used with multiprocessing
1350+
using configuration files. The configurations are fairly simple, but serve to
1351+
illustrate how more complex ones could be implemented in a real multiprocessing
1352+
scenario.
1353+
1354+
In the example, the main process spawns a listener process and some worker
1355+
processes. Each of the main process, the listener and the workers have three
1356+
separate configurations (the workers all share the same configuration). We can
1357+
see logging in the main process, how the workers log to a QueueHandler and how
1358+
the listener implements a QueueListener and a more complex logging
1359+
configuration, and arranges to dispatch events received via the queue to the
1360+
handlers specified in the configuration. Note that these configurations are
1361+
purely illustrative, but you should be able to adapt this example to your own
1362+
scenario.
1363+
1364+
Here's the script - the docstrings and the comments hopefully explain how it
1365+
works::
1366+
1367+
import logging
1368+
import logging.config
1369+
import logging.handlers
1370+
from multiprocessing import Process, Queue, Event, current_process
1371+
import os
1372+
import random
1373+
import time
1374+
1375+
class MyHandler(object):
1376+
"""
1377+
A simple handler for logging events. It runs in the listener process and
1378+
dispatches events to loggers based on the name in the received record,
1379+
which then get dispatched, by the logging system, to the handlers
1380+
configured for those records.
1381+
"""
1382+
def handle(self, record):
1383+
logger = logging.getLogger(record.name)
1384+
# The process name is transformed just to show that it's the listener
1385+
# doing the logging to files and console
1386+
record.processName = '%s (for %s)' % (current_process().name, record.processName)
1387+
logger.handle(record)
1388+
1389+
def listener_process(q, stop_event, config):
1390+
"""
1391+
This could be done in the main process, but is just done in a separate
1392+
process for illustrative purposes.
1393+
1394+
This initialises logging according to the specified configuration,
1395+
starts the listener and waits for the main process to signal completion
1396+
via the event. The listener is then stopped, and the process exits.
1397+
"""
1398+
logging.config.dictConfig(config)
1399+
listener = logging.handlers.QueueListener(q, MyHandler())
1400+
listener.start()
1401+
if os.name == 'posix':
1402+
# On POSIX, the setup logger will have been configured in the
1403+
# parent process, but should have been disabled following the
1404+
# dictConfig call.
1405+
# On Windows, since fork isn't used, the setup logger won't
1406+
# exist in the child, so it would be created and the message
1407+
# would appear - hence the "if posix" clause.
1408+
logger = logging.getLogger('setup')
1409+
logger.critical('Should not appear, because of disabled logger ...')
1410+
stop_event.wait()
1411+
listener.stop()
1412+
1413+
def worker_process(config):
1414+
"""
1415+
A number of these are spawned for the purpose of illustration. In
1416+
practice, they could be a heterogenous bunch of processes rather than
1417+
ones which are identical to each other.
1418+
1419+
This initialises logging according to the specified configuration,
1420+
and logs a hundred messages with random levels to randomly selected
1421+
loggers.
1422+
1423+
A small sleep is added to allow other processes a chance to run. This
1424+
is not strictly needed, but it mixes the output from the different
1425+
processes a bit more than if it's left out.
1426+
"""
1427+
logging.config.dictConfig(config)
1428+
levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR,
1429+
logging.CRITICAL]
1430+
loggers = ['foo', 'foo.bar', 'foo.bar.baz',
1431+
'spam', 'spam.ham', 'spam.ham.eggs']
1432+
if os.name == 'posix':
1433+
# On POSIX, the setup logger will have been configured in the
1434+
# parent process, but should have been disabled following the
1435+
# dictConfig call.
1436+
# On Windows, since fork isn't used, the setup logger won't
1437+
# exist in the child, so it would be created and the message
1438+
# would appear - hence the "if posix" clause.
1439+
logger = logging.getLogger('setup')
1440+
logger.critical('Should not appear, because of disabled logger ...')
1441+
for i in range(100):
1442+
lvl = random.choice(levels)
1443+
logger = logging.getLogger(random.choice(loggers))
1444+
logger.log(lvl, 'Message no. %d', i)
1445+
time.sleep(0.01)
1446+
1447+
def main():
1448+
q = Queue()
1449+
# The main process gets a simple configuration which prints to the console.
1450+
config_initial = {
1451+
'version': 1,
1452+
'formatters': {
1453+
'detailed': {
1454+
'class': 'logging.Formatter',
1455+
'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s'
1456+
}
1457+
},
1458+
'handlers': {
1459+
'console': {
1460+
'class': 'logging.StreamHandler',
1461+
'level': 'INFO',
1462+
},
1463+
},
1464+
'root': {
1465+
'level': 'DEBUG',
1466+
'handlers': ['console']
1467+
},
1468+
}
1469+
# The worker process configuration is just a QueueHandler attached to the
1470+
# root logger, which allows all messages to be sent to the queue.
1471+
# We disable existing loggers to disable the "setup" logger used in the
1472+
# parent process. This is needed on POSIX because the logger will
1473+
# be there in the child following a fork().
1474+
config_worker = {
1475+
'version': 1,
1476+
'disable_existing_loggers': True,
1477+
'handlers': {
1478+
'queue': {
1479+
'class': 'logging.handlers.QueueHandler',
1480+
'queue': q,
1481+
},
1482+
},
1483+
'root': {
1484+
'level': 'DEBUG',
1485+
'handlers': ['queue']
1486+
},
1487+
}
1488+
# The listener process configuration shows that the full flexibility of
1489+
# logging configuration is available to dispatch events to handlers however
1490+
# you want.
1491+
# We disable existing loggers to disable the "setup" logger used in the
1492+
# parent process. This is needed on POSIX because the logger will
1493+
# be there in the child following a fork().
1494+
config_listener = {
1495+
'version': 1,
1496+
'disable_existing_loggers': True,
1497+
'formatters': {
1498+
'detailed': {
1499+
'class': 'logging.Formatter',
1500+
'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s'
1501+
},
1502+
'simple': {
1503+
'class': 'logging.Formatter',
1504+
'format': '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s'
1505+
}
1506+
},
1507+
'handlers': {
1508+
'console': {
1509+
'class': 'logging.StreamHandler',
1510+
'level': 'INFO',
1511+
'formatter': 'simple',
1512+
},
1513+
'file': {
1514+
'class': 'logging.FileHandler',
1515+
'filename': 'mplog.log',
1516+
'mode': 'w',
1517+
'formatter': 'detailed',
1518+
},
1519+
'foofile': {
1520+
'class': 'logging.FileHandler',
1521+
'filename': 'mplog-foo.log',
1522+
'mode': 'w',
1523+
'formatter': 'detailed',
1524+
},
1525+
'errors': {
1526+
'class': 'logging.FileHandler',
1527+
'filename': 'mplog-errors.log',
1528+
'mode': 'w',
1529+
'level': 'ERROR',
1530+
'formatter': 'detailed',
1531+
},
1532+
},
1533+
'loggers': {
1534+
'foo': {
1535+
'handlers' : ['foofile']
1536+
}
1537+
},
1538+
'root': {
1539+
'level': 'DEBUG',
1540+
'handlers': ['console', 'file', 'errors']
1541+
},
1542+
}
1543+
# Log some initial events, just to show that logging in the parent works
1544+
# normally.
1545+
logging.config.dictConfig(config_initial)
1546+
logger = logging.getLogger('setup')
1547+
logger.info('About to create workers ...')
1548+
workers = []
1549+
for i in range(5):
1550+
wp = Process(target=worker_process, name='worker %d' % (i + 1),
1551+
args=(config_worker,))
1552+
workers.append(wp)
1553+
wp.start()
1554+
logger.info('Started worker: %s', wp.name)
1555+
logger.info('About to create listener ...')
1556+
stop_event = Event()
1557+
lp = Process(target=listener_process, name='listener',
1558+
args=(q, stop_event, config_listener))
1559+
lp.start()
1560+
logger.info('Started listener')
1561+
# We now hang around for the workers to finish their work.
1562+
for wp in workers:
1563+
wp.join()
1564+
# Workers all done, listening can now stop.
1565+
# Logging in the parent still works normally.
1566+
logger.info('Telling listener to stop ...')
1567+
stop_event.set()
1568+
lp.join()
1569+
logger.info('All done.')
1570+
1571+
if __name__ == '__main__':
1572+
main()
1573+

0 commit comments

Comments
 (0)