From e3b23c2d816c478aabc1d0bfdfc3f07611dbe6af Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 16:55:49 +0100 Subject: [PATCH 01/24] Initial commit Add a Barrier object to asyncio synchronized primitives --- Lib/asyncio/locks.py | 173 +++++++++++++- Lib/test/test_asyncio/test_locks.py | 356 +++++++++++++++++++++++++++- 2 files changed, 527 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a7453fb1c772878..e5fcf9648de5d9d 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,6 +1,6 @@ """Synchronization primitives.""" -__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore') +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections @@ -418,3 +418,174 @@ def release(self): if self._value >= self._bound_value: raise ValueError('BoundedSemaphore released too many times') super().release() + + +# A barrier class. Inspired in part by the pthread_barrier_* api and +# the CyclicBarrier class from Java. See +# http://sourceware.org/pthreads-win32/manual/pthread_barrier_init.html and +# http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/ +# CyclicBarrier.html +# for information. +# We maintain two main states, 'filling' and 'draining' enabling the barrier +# to be cyclic. Tasks are not allowed into it until it has fully drained +# since the previous cycle. In addition, a 'resetting' state exists which is +# similar to 'draining' except that tasks leave with a BrokenBarrierError, +# and a 'broken' state in which all tasks get the exception. + +class Barrier(mixins._LoopBoundMixin): + """Asynchronous equivalent to threading.Barrier + + Implements a Barrier. + Useful for synchronizing a fixed number of tasks at known synchronization + points. Tasks block on 'wait()' and are simultaneously awoken once they + have all made that call. + """ + + def __init__(self, parties, action=None, *, loop=mixins._marker): + """Create a barrier, initialised to 'parties' tasks. + 'action' is a callable which, when supplied, will be called by one of + the tasks after they have all entered the barrier and just prior to + releasing them all. + """ + super().__init__(loop=loop) + if parties < 1: + raise ValueError('parties must be > 0') + + self._waiting = Event() # used notify all waiting tasks + self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._action = action + self._parties = parties + self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._count = 0 # count waiting tasks + + def __repr__(self): + res = super().__repr__() + _wait = 'set' if self._waiting.is_set() else 'unset' + _block = 'set' if self._blocking.is_set() else 'unset' + extra = f'{_wait}, count:{self._count}/{self._parties}, {_block}, state:{self._state}' + return f'<{res[1:-1]} [{extra}]>' + + async def wait(self): + """Wait for the barrier. + When the specified number of tasks have started waiting, they are all + simultaneously awoken. If an 'action' was provided for the barrier, one + of the tasks will have executed that callback prior to returning. + Returns an individual index number from 0 to 'parties-1'. + """ + await self._block() # Block while the barrier drains or resets. + index = self._count + self._count += 1 + try: + if index + 1 == self._parties: + # We release the barrier + self._release() + else: + # We wait until someone releases us + await self._wait() + return index + finally: + self._count -= 1 + # Wake up any tasks waiting for barrier to drain. + self._exit() + + # Block until the barrier is ready for us, or raise an exception + # if it is broken. + async def _block (self): + if self._state in (-1, 1): + # It is draining or resetting, wait until done + await self._blocking.wait() + + #see if the barrier is in a broken state + if self._state < 0: + raise BrokenBarrierError + assert self._state == 0, repr(self) + + # Optionally run the 'action' and release the tasks waiting + # in the barrier. + def _release(self): + try: + if self._action: + self._action() + # enter draining state + self._state = 1 + self._blocking.clear() + self._waiting.set() + except: + #an exception during the _action handler. Break and reraise + self._state = -2 + self._blocking.clear() + self._waiting.set() + raise + + # Wait in the barrier until we are released. Raise an exception + # if the barrier is reset or broken. + async def _wait(self): + await self._waiting.wait() + # no timeout so + if self._state < 0: + raise BrokenBarrierError + assert self._state == 1, repr(self) + + # If we are the last tasks to exit the barrier, signal any tasks + # waiting for the barrier to drain. + def _exit(self): + if self._count == 0: + if self._state == 1: + self._state = 0 + elif self._state == -1: + self._state = 0 + self._waiting.clear() + self._blocking.set() + + # async def reset(self): + def reset(self): + """Reset the barrier to the initial state. + Any tasks currently waiting will get the BrokenBarrier exception + raised. + """ + if self._count > 0: + if self._state in (0, 1): + #reset the barrier, waking up tasks + self._state = -1 + elif self._state == -2: + #was broken, set it to reset state + #which clears when the last tasks exits + self._state = -1 + self._waiting.set() + self._blocking.clear() + else: + self._state = 0 + + + # async def abort(self): + def abort(self): + """Place the barrier into a 'broken' state. + Useful in case of error. Any currently waiting tasks and tasks + attempting to 'wait()' will have BrokenBarrierError raised. + """ + self._state = -2 + self._waiting.set() + self._blocking.clear() + + @property + def parties(self): + """Return the number of tasks required to trip the barrier.""" + return self._parties + + @property + def n_waiting(self): + """Return the number of tasks currently waiting at the barrier.""" + # We don't need synchronization here since this is an ephemeral result + # anyway. It returns the correct value in the steady state. + if self._state == 0: + return self._count + return 0 + + @property + def broken(self): + """Return True if the barrier is in a broken state.""" + return self._state == -2 + +# exception raised by the Barrier class +class BrokenBarrierError(RuntimeError): + pass \ No newline at end of file diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 6194cd06176dac3..824535a18cb168e 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -6,11 +6,13 @@ import asyncio from test.test_asyncio import utils as test_utils +import functools STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?(, count:\d+/\d+)?(, state:^-?\d)?' + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier object r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -68,6 +70,16 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) + # Barrier object has a positional paramater + # so check alone + cls = asyncio.Barrier + with self.assertRaisesRegex( + TypeError, + rf'As of 3.10, the \*loop\* parameter was removed from ' + rf'{cls.__name__}\(\) since it is no longer necessary' + ): + cls(1, loop=self.loop) + def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) @@ -928,5 +940,347 @@ def test_release_no_waiters(self): self.assertFalse(sem.locked()) +class BarrierTests(test_utils.TestCase): + """ + Tests for Barrier objects. + """ + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.N = 5 + + async def run_coros(self, n, coro): + tasks = [self.loop.create_task(coro()) for _ in range(n)] + res = await asyncio.gather(*tasks) + return res, tasks + + def test_repr(self): + async def coro(): + try: + i = await barrier.wait() + except: + ... + else: + return await asyncio.sleep(1, True) + + self.N = 999 + barrier = asyncio.Barrier(self.N) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + + nb_waits = 3 + tasks = [] + for _ in range(nb_waits): + tasks.append(self.loop.create_task(coro())) + test_utils.run_briefly(self.loop) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + + barrier.reset() + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue("[set," in repr(barrier)) + self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('unset, state:-1]>')) + test_utils.run_briefly(self.loop) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(f"count:0/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('set, state:0]>')) + test_utils.run_briefly(self.loop) + + barrier.abort() + test_utils.run_briefly(self.loop) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"0/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith(':-2]>')) + + def test_init(self): + self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) + self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) + + def test_wait(self): + async def coro(result): + r = await barrier.wait() + result.append(r) + + barrier = asyncio.Barrier(self.N) + results = [] + _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + self.assertEqual(sum(results), sum(range(self.N))) + self.assertFalse(barrier.broken) + + def test_wait_task_multi(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + self.assertEqual(barrier.n_waiting, 0) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_task(self): + self.N = 1 + barrier = asyncio.Barrier(self.N) + r1 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(r1, r2) + self.assertFalse(barrier.broken) + + def test_wait_step_by_step(self): + async def coro(result, value): + await barrier.wait() + result.append(value) + + results = [] + barrier = asyncio.Barrier(self.N) + value = 1 + tasks = [] + for n in range(self.N-1): + tasks.append(self.loop.create_task(coro(results, value))) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, n+1) + + tasks.append(self.loop.create_task(coro(results, value))) + test_utils.run_briefly(self.loop) + self.assertEqual(results, [value]* self.N) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(all(t.done() for t in tasks)) + self.assertTrue(all(t.result() is None for t in tasks)) + self.assertFalse(barrier.broken) + + def test_wait_multi_return(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + await barrier.wait() + results1.append(True) + i = await barrier.wait() + results2.append(True) + results3.append(i) + return i + + barrier = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results1), self.N) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + self.assertEqual(sum(res), sum(results3)) + + def test_barrier(self, multipass=1, nn=5): + results = [[] for _ in range(multipass)] + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + barrier = asyncio.Barrier(nn) + for i in range(multipass): + _ = [self.loop.create_task(coro(results[i], value)) for value in range(nn)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results[i]), nn) + self.assertEqual(sum(results[i]), sum(range(nn))) + if i > 0: + self.assertEqual(sum(results[i]), sum(results[i-1])) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) + + def test_barrier_multipass(self): + self.test_barrier(10) + + def test_action_callback(self): + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + result = [] + result1 = [] + value = 1 + barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) + _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), self.N) + self.assertTrue(all(result1)) + self.assertEqual(len(result), self.N) + self.assertEqual(sum(result), self.N*value) + self.assertFalse(barrier.broken) + + def test_action_callback_n(self): + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + result = [] + result1 = [] + value = 1 + barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) + _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), 1) + self.assertTrue(result1[0]) + self.assertEqual(len(result), self.N) + self.assertEqual(sum(result), value*self.N) + self.assertFalse(barrier.broken) + + def test_action_callback_error(self): + results1 = [] + results2 = [] + results3 = [] + def raise_except(e): + raise e + + async def coro(e): + try: + ret = await barrier.wait() + except e: + results1.append(False) + except: + results2.append(True) + else: + results3.append(None) + + barrier = asyncio.Barrier(self.N, lambda: raise_except(ZeroDivisionError)) + _ = [self.loop.create_task(coro(ZeroDivisionError)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + self.assertEqual(len(results1), 1) + self.assertFalse(results1[0]) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(len(results3), 0) + self.assertTrue(barrier.broken) + + def test_reset(self): + results1 = [] + results2 = [] + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results1.append(True) + finally: + await barrier.wait() + results2.append(True) + + async def coro2(): + await barrier.wait() + results2.append(True) + + + barrier = asyncio.Barrier(self.N) + _ = [self.loop.create_task(coro()) for _ in range(self.N-1)] + test_utils.run_briefly(self.loop) + barrier.reset() + test_utils.run_briefly(self.loop) + self.loop.create_task(coro2()) + test_utils.run_briefly(self.loop) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), self.N) + + def test_reset_multi_wait(self): + results1 = [] + results2 = [] + results3 = [] + alls = [] + async def coro(): + i = await barrier1.wait() + alls.append(i) + if len(alls) == self.N: # i == self.N//2: + barrier.reset() + else: + try: + await barrier.wait() + results1.append(True) + except Exception as e: + results2.append(True) + + # Now, pass the barrier again + await barrier1.wait() + results3.append(True) + + barrier = asyncio.Barrier(self.N) + barrier1 = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + def test_abort(self): + results1 = [] + results2 = [] + async def coro(): + try: + i = await barrier.wait() + if i == self.N//2: + raise RuntimeError + await barrier.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + barrier = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertTrue(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + + def test_abort_and_reset(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + try: + i = await barrier.wait() + if i == self.N//2: + raise RuntimeError + await barrier.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + i = await barrier2.wait() + if i == self.N//2: + barrier.reset() + await barrier2.wait() + await barrier.wait() + results3.append(True) + + barrier = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + if __name__ == '__main__': unittest.main() From bc3d17a45bc554a2bad15659690368ec0ee4db9e Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 21:04:37 +0100 Subject: [PATCH 02/24] make patchcheck Change after run make patchcheck --- Lib/asyncio/locks.py | 18 ++++++++-------- Lib/test/test_asyncio/test_locks.py | 32 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e5fcf9648de5d9d..4bc69100ed225d8 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -433,7 +433,7 @@ def release(self): # and a 'broken' state in which all tasks get the exception. class Barrier(mixins._LoopBoundMixin): - """Asynchronous equivalent to threading.Barrier + """Asynchronous equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization @@ -445,18 +445,18 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. 'action' is a callable which, when supplied, will be called by one of the tasks after they have all entered the barrier and just prior to - releasing them all. + releasing them all. """ super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._blocking = Event() # used block tasks while wainting tasks are draining or broken self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken - self._count = 0 # count waiting tasks + self._count = 0 # count waiting tasks def __repr__(self): res = super().__repr__() @@ -494,7 +494,7 @@ async def _block (self): if self._state in (-1, 1): # It is draining or resetting, wait until done await self._blocking.wait() - + #see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError @@ -515,7 +515,7 @@ def _release(self): self._state = -2 self._blocking.clear() self._waiting.set() - raise + raise # Wait in the barrier until we are released. Raise an exception # if the barrier is reset or broken. @@ -529,7 +529,7 @@ async def _wait(self): # If we are the last tasks to exit the barrier, signal any tasks # waiting for the barrier to drain. def _exit(self): - if self._count == 0: + if self._count == 0: if self._state == 1: self._state = 0 elif self._state == -1: @@ -543,7 +543,7 @@ def reset(self): Any tasks currently waiting will get the BrokenBarrier exception raised. """ - if self._count > 0: + if self._count > 0: if self._state in (0, 1): #reset the barrier, waking up tasks self._state = -1 @@ -588,4 +588,4 @@ def broken(self): # exception raised by the Barrier class class BrokenBarrierError(RuntimeError): - pass \ No newline at end of file + pass diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 824535a18cb168e..42ef44db2d37589 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -70,8 +70,8 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) - # Barrier object has a positional paramater - # so check alone + # Barrier object has a positional paramater + # so check alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, @@ -947,7 +947,7 @@ class BarrierTests(test_utils.TestCase): def setUp(self): super().setUp() self.loop = self.new_test_loop() - self.N = 5 + self.N = 5 async def run_coros(self, n, coro): tasks = [self.loop.create_task(coro()) for _ in range(n)] @@ -969,8 +969,8 @@ async def coro(): self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(repr(barrier).endswith('unset, state:0]>')) - - nb_waits = 3 + + nb_waits = 3 tasks = [] for _ in range(nb_waits): tasks.append(self.loop.create_task(coro())) @@ -1002,16 +1002,16 @@ def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait(self): + def test_wait_more(self): async def coro(result): r = await barrier.wait() result.append(r) - + barrier = asyncio.Barrier(self.N) results = [] _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] test_utils.run_briefly(self.loop) - self.assertEqual(sum(results), sum(range(self.N))) + self.assertEqual(sum(results), sum(range(self.N))) self.assertFalse(barrier.broken) def test_wait_task_multi(self): @@ -1029,7 +1029,7 @@ def test_wait_task_multi(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_task(self): + def test_wait_task_one(self): self.N = 1 barrier = asyncio.Barrier(self.N) r1 = self.loop.run_until_complete(barrier.wait()) @@ -1115,7 +1115,7 @@ async def coro(result, value): value = 1 barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) self.assertEqual(len(result1), self.N) self.assertTrue(all(result1)) @@ -1134,7 +1134,7 @@ async def coro(result, value): value = 1 barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) self.assertEqual(len(result1), 1) self.assertTrue(result1[0]) @@ -1173,9 +1173,9 @@ def test_reset(self): results1 = [] results2 = [] async def coro(): - try: + try: await barrier.wait() - except asyncio.BrokenBarrierError: + except asyncio.BrokenBarrierError: results1.append(True) finally: await barrier.wait() @@ -1217,7 +1217,7 @@ async def coro(): # Now, pass the barrier again await barrier1.wait() results3.append(True) - + barrier = asyncio.Barrier(self.N) barrier1 = asyncio.Barrier(self.N) res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) @@ -1235,7 +1235,7 @@ async def coro(): if i == self.N//2: raise RuntimeError await barrier.wait() - results1.append(True) + results1.append(True) except asyncio.BrokenBarrierError: results2.append(True) except RuntimeError: @@ -1272,7 +1272,7 @@ async def coro(): await barrier2.wait() await barrier.wait() results3.append(True) - + barrier = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) From a8b7b1acc1a991d5985c2ca0515ca8215fb2d56f Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 21:06:51 +0100 Subject: [PATCH 03/24] make patchcheck run make patchcheck --- Doc/c-api/exceptions.rst.bak | 1054 +++++++++++++ Doc/c-api/memory.rst.bak | 642 ++++++++ Doc/faq/design.rst.bak | 759 +++++++++ Doc/howto/descriptor.rst.bak | 1575 +++++++++++++++++++ Doc/library/_thread.rst.bak | 215 +++ Doc/library/ast.rst.bak | 1993 ++++++++++++++++++++++++ Doc/library/asyncio-api-index.rst.bak | 221 +++ Doc/library/collections.rst.bak | 1278 +++++++++++++++ Doc/library/dataclasses.rst.bak | 595 +++++++ Doc/library/gc.rst.bak | 322 ++++ Doc/library/importlib.metadata.rst.bak | 262 ++++ Doc/library/logging.rst.bak | 1350 ++++++++++++++++ Doc/library/readline.rst.bak | 361 +++++ Doc/library/sqlite3.rst.bak | 1094 +++++++++++++ Doc/library/statistics.rst.bak | 879 +++++++++++ Doc/library/tempfile.rst.bak | 374 +++++ Doc/library/venv.rst.bak | 496 ++++++ Doc/library/xml.sax.handler.rst.bak | 463 ++++++ Doc/library/zipimport.rst.bak | 209 +++ Doc/whatsnew/3.10.rst.bak | 1315 ++++++++++++++++ 20 files changed, 15457 insertions(+) create mode 100644 Doc/c-api/exceptions.rst.bak create mode 100644 Doc/c-api/memory.rst.bak create mode 100644 Doc/faq/design.rst.bak create mode 100644 Doc/howto/descriptor.rst.bak create mode 100644 Doc/library/_thread.rst.bak create mode 100644 Doc/library/ast.rst.bak create mode 100644 Doc/library/asyncio-api-index.rst.bak create mode 100644 Doc/library/collections.rst.bak create mode 100644 Doc/library/dataclasses.rst.bak create mode 100644 Doc/library/gc.rst.bak create mode 100644 Doc/library/importlib.metadata.rst.bak create mode 100644 Doc/library/logging.rst.bak create mode 100644 Doc/library/readline.rst.bak create mode 100644 Doc/library/sqlite3.rst.bak create mode 100644 Doc/library/statistics.rst.bak create mode 100644 Doc/library/tempfile.rst.bak create mode 100644 Doc/library/venv.rst.bak create mode 100644 Doc/library/xml.sax.handler.rst.bak create mode 100644 Doc/library/zipimport.rst.bak create mode 100644 Doc/whatsnew/3.10.rst.bak diff --git a/Doc/c-api/exceptions.rst.bak b/Doc/c-api/exceptions.rst.bak new file mode 100644 index 000000000000000..4e99a0167a632d7 --- /dev/null +++ b/Doc/c-api/exceptions.rst.bak @@ -0,0 +1,1054 @@ +.. highlight:: c + + +.. _exceptionhandling: + +****************** +Exception Handling +****************** + +The functions described in this chapter will let you handle and raise Python +exceptions. It is important to understand some of the basics of Python +exception handling. It works somewhat like the POSIX :c:data:`errno` variable: +there is a global indicator (per thread) of the last error that occurred. Most +C API functions don't clear this on success, but will set it to indicate the +cause of the error on failure. Most C API functions also return an error +indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` +if they return an integer (exception: the :c:func:`PyArg_\*` functions +return ``1`` for success and ``0`` for failure). + +Concretely, the error indicator consists of three object pointers: the +exception's type, the exception's value, and the traceback object. Any +of those pointers can be ``NULL`` if non-set (although some combinations are +forbidden, for example you can't have a non-``NULL`` traceback if the exception +type is ``NULL``). + +When a function must fail because some function it called failed, it generally +doesn't set the error indicator; the function it called already set it. It is +responsible for either handling the error and clearing the exception or +returning after cleaning up any resources it holds (such as object references or +memory allocations); it should *not* continue normally if it is not prepared to +handle the error. If returning due to an error, it is important to indicate to +the caller that an error has been set. If the error is not handled or carefully +propagated, additional calls into the Python/C API may not behave as intended +and may fail in mysterious ways. + +.. note:: + The error indicator is **not** the result of :func:`sys.exc_info()`. + The former corresponds to an exception that is not yet caught (and is + therefore still propagating), while the latter returns an exception after + it is caught (and has therefore stopped propagating). + + +Printing and clearing +===================== + + +.. c:function:: void PyErr_Clear() + + Clear the error indicator. If the error indicator is not set, there is no + effect. + + +.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) + + Print a standard traceback to ``sys.stderr`` and clear the error indicator. + **Unless** the error is a ``SystemExit``, in that case no traceback is + printed and the Python process will exit with the error code specified by + the ``SystemExit`` instance. + + Call this function **only** when the error indicator is set. Otherwise it + will cause a fatal error! + + If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, + :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the + type, value and traceback of the printed exception, respectively. + + +.. c:function:: void PyErr_Print() + + Alias for ``PyErr_PrintEx(1)``. + + +.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + + Call :func:`sys.unraisablehook` using the current exception and *obj* + argument. + + This utility function prints a warning message to ``sys.stderr`` when an + exception has been set but it is impossible for the interpreter to actually + raise the exception. It is used, for example, when an exception occurs in an + :meth:`__del__` method. + + The function is called with a single argument *obj* that identifies the context + in which the unraisable exception occurred. If possible, + the repr of *obj* will be printed in the warning message. + + An exception must be set when calling this function. + + +Raising exceptions +================== + +These functions help you set the current thread's error indicator. +For convenience, some of these functions will always return a +``NULL`` pointer for use in a ``return`` statement. + + +.. c:function:: void PyErr_SetString(PyObject *type, const char *message) + + This is the most common way to set the error indicator. The first argument + specifies the exception type; it is normally one of the standard exceptions, + e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + The second argument is an error message; it is decoded from ``'utf-8``'. + + +.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) + + This function is similar to :c:func:`PyErr_SetString` but lets you specify an + arbitrary Python object for the "value" of the exception. + + +.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) + + This function sets the error indicator and returns ``NULL``. *exception* + should be a Python exception class. The *format* and subsequent + parameters help format the error message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded + string. + + +.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) + + Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather + than a variable number of arguments. + + .. versionadded:: 3.5 + + +.. c:function:: void PyErr_SetNone(PyObject *type) + + This is a shorthand for ``PyErr_SetObject(type, Py_None)``. + + +.. c:function:: int PyErr_BadArgument() + + This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where + *message* indicates that a built-in operation was invoked with an illegal + argument. It is mostly for internal use. + + +.. c:function:: PyObject* PyErr_NoMemory() + + This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` + so an object allocation function can write ``return PyErr_NoMemory();`` when it + runs out of memory. + + +.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) + + .. index:: single: strerror() + + This is a convenience function to raise an exception when a C library function + has returned an error and set the C variable :c:data:`errno`. It constructs a + tuple object whose first item is the integer :c:data:`errno` value and whose + second item is the corresponding error message (gotten from :c:func:`strerror`), + and then calls ``PyErr_SetObject(type, object)``. On Unix, when the + :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, + this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, + leaves it set to that. The function always returns ``NULL``, so a wrapper + function around a system call can write ``return PyErr_SetFromErrno(type);`` + when the system call returns an error. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) + + Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if + *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as + a third parameter. In the case of :exc:`OSError` exception, + this is used to define the :attr:`filename` attribute of the + exception instance. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second + filename object, for raising errors when a function that takes two filenames + fails. + + .. versionadded:: 3.4 + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename + is given as a C string. *filename* is decoded from the :term:`filesystem + encoding and error handler`. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) + + This is a convenience function to raise :exc:`WindowsError`. If called with + *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` + is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve + the Windows description of error code given by *ierr* or :c:func:`GetLastError`, + then it constructs a tuple object whose first item is the *ierr* value and whose + second item is the corresponding error message (gotten from + :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + object)``. This function always returns ``NULL``. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) + + Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter + specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the + filename is given as a C string. *filename* is decoded from the filesystem + encoding (:func:`os.fsdecode`). + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an + additional parameter specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) + + Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, + but accepts a second filename object. + + .. availability:: Windows. + + .. versionadded:: 3.4 + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional + parameter specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) + + This is a convenience function to raise :exc:`ImportError`. *msg* will be + set as the exception's message string. *name* and *path*, both of which can + be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` + and ``path`` attributes. + + .. versionadded:: 3.3 + + +.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) + + Set file, line, and offset information for the current exception. If the + current exception is not a :exc:`SyntaxError`, then it sets additional + attributes, which make the exception printing subsystem think the exception + is a :exc:`SyntaxError`. + + .. versionadded:: 3.4 + + +.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) + + Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string + decoded from the :term:`filesystem encoding and error handler`. + + .. versionadded:: 3.2 + + +.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) + + Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is + omitted. + + +.. c:function:: void PyErr_BadInternalCall() + + This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, + where *message* indicates that an internal operation (e.g. a Python/C API + function) was invoked with an illegal argument. It is mostly for internal + use. + + +Issuing warnings +================ + +Use these functions to issue warnings from C code. They mirror similar +functions exported by the Python :mod:`warnings` module. They normally +print a warning message to *sys.stderr*; however, it is +also possible that the user has specified that warnings are to be turned into +errors, and in that case they will raise an exception. It is also possible that +the functions raise an exception because of a problem with the warning machinery. +The return value is ``0`` if no exception is raised, or ``-1`` if an exception +is raised. (It is not possible to determine whether a warning message is +actually printed, nor what the reason is for the exception; this is +intentional.) If an exception is raised, the caller should do its normal +exception handling (for example, :c:func:`Py_DECREF` owned references and return +an error value). + +.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) + + Issue a warning message. The *category* argument is a warning category (see + below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a + positive number giving a number of stack frames; the warning will be issued from + the currently executing line of code in that stack frame. A *stack_level* of 1 + is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, + and so forth. + + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standardwarningcategories`. + + For information about warning control, see the documentation for the + :mod:`warnings` module and the :option:`-W` option in the command line + documentation. There is no C API for warning control. + +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) + + Much like :c:func:`PyErr_SetImportError` but this function allows for + specifying a subclass of :exc:`ImportError` to raise. + + .. versionadded:: 3.6 + + +.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) + + Issue a warning message with explicit control over all warning attributes. This + is a straightforward wrapper around the Python function + :func:`warnings.warn_explicit`, see there for more information. The *module* + and *registry* arguments may be set to ``NULL`` to get the default effect + described there. + + .. versionadded:: 3.4 + + +.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) + + Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and + *module* are UTF-8 encoded strings, and *filename* is decoded from the + :term:`filesystem encoding and error handler`. + + +.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnEx`, but use + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + +.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnFormat`, but *category* is + :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + + .. versionadded:: 3.6 + + +Querying the error indicator +============================ + +.. c:function:: PyObject* PyErr_Occurred() + + Test whether the error indicator is set. If set, return the exception *type* + (the first argument to the last call to one of the :c:func:`PyErr_Set\*` + functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not + own a reference to the return value, so you do not need to :c:func:`Py_DECREF` + it. + + The caller must hold the GIL. + + .. note:: + + Do not compare the return value to a specific exception; use + :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could + easily fail since the exception may be an instance instead of a class, in the + case of a class exception, or it may be a subclass of the expected exception.) + + +.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) + + Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This + should only be called when an exception is actually set; a memory access + violation will occur if no exception has been raised. + + +.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) + + Return true if the *given* exception matches the exception type in *exc*. If + *exc* is a class object, this also returns true when *given* is an instance + of a subclass. If *exc* is a tuple, all exception types in the tuple (and + recursively in subtuples) are searched for a match. + + +.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the error indicator into three variables whose addresses are passed. + If the error indicator is not set, set all three variables to ``NULL``. If it is + set, it will be cleared and you own a reference to each object retrieved. The + value and traceback object may be ``NULL`` even when the type object is not. + + .. note:: + + This function is normally only used by code that needs to catch exceptions or + by code that needs to save and restore the error indicator temporarily, e.g.:: + + { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + + /* ... code that might produce other errors ... */ + + PyErr_Restore(type, value, traceback); + } + + +.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) + + Set the error indicator from the three objects. If the error indicator is + already set, it is cleared first. If the objects are ``NULL``, the error + indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or + traceback. The exception type should be a class. Do not pass an invalid + exception type or value. (Violating these rules will cause subtle problems + later.) This call takes away a reference to each object: you must own a + reference to each object before the call and after the call you no longer own + these references. (If you don't understand this, don't use this function. I + warned you.) + + .. note:: + + This function is normally only used by code that needs to save and restore the + error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current + error indicator. + + +.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) + + Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below + can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance. + + .. note:: + + This function *does not* implicitly set the ``__traceback__`` + attribute on the exception value. If setting the traceback + appropriately is desired, the following additional snippet is needed:: + + if (tb != NULL) { + PyException_SetTraceback(val, tb); + } + + +.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the exception info, as known from ``sys.exc_info()``. This refers + to an exception that was *already caught*, not to an exception that was + freshly raised. Returns new references for the three objects, any of which + may be ``NULL``. Does not modify the exception info state. + + .. note:: + + This function is not normally used by code that wants to handle exceptions. + Rather, it can be used when code needs to save and restore the exception + state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the + exception state. + + .. versionadded:: 3.3 + + +.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) + + Set the exception info, as known from ``sys.exc_info()``. This refers + to an exception that was *already caught*, not to an exception that was + freshly raised. This function steals the references of the arguments. + To clear the exception state, pass ``NULL`` for all three arguments. + For general rules about the three arguments, see :c:func:`PyErr_Restore`. + + .. note:: + + This function is not normally used by code that wants to handle exceptions. + Rather, it can be used when code needs to save and restore the exception + state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception + state. + + .. versionadded:: 3.3 + + +Signal Handling +=============== + + +.. c:function:: int PyErr_CheckSignals() + + .. index:: + module: signal + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + This function interacts with Python's signal handling. It checks whether a + signal has been sent to the processes and if so, invokes the corresponding + signal handler. If the :mod:`signal` module is supported, this can invoke a + signal handler written in Python. In all cases, the default effect for + :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an + exception is raised the error indicator is set and the function returns ``-1``; + otherwise the function returns ``0``. The error indicator may or may not be + cleared if it was previously set. + + +.. c:function:: void PyErr_SetInterrupt() + + .. index:: + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. + +.. c:function:: int PySignal_SetWakeupFd(int fd) + + This utility function specifies a file descriptor to which the signal number + is written as a single byte whenever a signal is received. *fd* must be + non-blocking. It returns the previous such file descriptor. + + The value ``-1`` disables the feature; this is the initial state. + This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any + error checking. *fd* should be a valid file descriptor. The function should + only be called from the main thread. + + .. versionchanged:: 3.5 + On Windows, the function now also supports socket handles. + + +Exception Classes +================= + +.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) + + This utility function creates and returns a new exception class. The *name* + argument must be the name of the new exception, a C string of the form + ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. + This creates a class object derived from :exc:`Exception` (accessible in C as + :c:data:`PyExc_Exception`). + + The :attr:`__module__` attribute of the new class is set to the first part (up + to the last dot) of the *name* argument, and the class name is set to the last + part (after the last dot). The *base* argument can be used to specify alternate + base classes; it can either be only one class or a tuple of classes. The *dict* + argument can be used to specify a dictionary of class variables and methods. + + +.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) + + Same as :c:func:`PyErr_NewException`, except that the new exception class can + easily be given a docstring: If *doc* is non-``NULL``, it will be used as the + docstring for the exception class. + + .. versionadded:: 3.2 + + +Exception Objects +================= + +.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) + + Return the traceback associated with the exception as a new reference, as + accessible from Python through :attr:`__traceback__`. If there is no + traceback associated, this returns ``NULL``. + + +.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) + + Set the traceback associated with the exception to *tb*. Use ``Py_None`` to + clear it. + + +.. c:function:: PyObject* PyException_GetContext(PyObject *ex) + + Return the context (another exception instance during whose handling *ex* was + raised) associated with the exception as a new reference, as accessible from + Python through :attr:`__context__`. If there is no context associated, this + returns ``NULL``. + + +.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) + + Set the context associated with the exception to *ctx*. Use ``NULL`` to clear + it. There is no type check to make sure that *ctx* is an exception instance. + This steals a reference to *ctx*. + + +.. c:function:: PyObject* PyException_GetCause(PyObject *ex) + + Return the cause (either an exception instance, or :const:`None`, + set by ``raise ... from ...``) associated with the exception as a new + reference, as accessible from Python through :attr:`__cause__`. + + +.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) + + Set the cause associated with the exception to *cause*. Use ``NULL`` to clear + it. There is no type check to make sure that *cause* is either an exception + instance or :const:`None`. This steals a reference to *cause*. + + :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + + +.. _unicodeexceptions: + +Unicode Exception Objects +========================= + +The following functions are used to create and modify Unicode exceptions from C. + +.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + +.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. + +.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeTranslateError` object with the attributes *object*, + *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. + + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) + + Return the *encoding* attribute of the given exception object. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) + + Return the *object* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) + + Get the *start* attribute of the given exception object and place it into + *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) + + Set the *start* attribute of the given exception object to *start*. Return + ``0`` on success, ``-1`` on failure. + +.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) + + Get the *end* attribute of the given exception object and place it into + *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) + + Set the *end* attribute of the given exception object to *end*. Return ``0`` + on success, ``-1`` on failure. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) + + Return the *reason* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) + + Set the *reason* attribute of the given exception object to *reason*. Return + ``0`` on success, ``-1`` on failure. + + +.. _recursion: + +Recursion Control +================= + +These two functions provide a way to perform safe recursive calls at the C +level, both in the core and in extension modules. They are needed if the +recursive code does not necessarily invoke Python code (which tracks its +recursion depth automatically). +They are also not needed for *tp_call* implementations +because the :ref:`call protocol ` takes care of recursion handling. + +.. c:function:: int Py_EnterRecursiveCall(const char *where) + + Marks a point where a recursive C-level call is about to be performed. + + If :const:`USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + sets a :exc:`MemoryError` and returns a nonzero value. + + The function then checks if the recursion limit is reached. If this is the + case, a :exc:`RecursionError` is set and a nonzero value is returned. + Otherwise, zero is returned. + + *where* should be a UTF-8 encoded string such as ``" in instance check"`` to + be concatenated to the :exc:`RecursionError` message caused by the recursion + depth limit. + + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +.. c:function:: void Py_LeaveRecursiveCall(void) + + Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each + *successful* invocation of :c:func:`Py_EnterRecursiveCall`. + + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires +special recursion handling. In addition to protecting the stack, +:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The +following two functions facilitate this functionality. Effectively, +these are the C equivalent to :func:`reprlib.recursive_repr`. + +.. c:function:: int Py_ReprEnter(PyObject *object) + + Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to + detect cycles. + + If the object has already been processed, the function returns a + positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation + should return a string object indicating a cycle. As examples, + :class:`dict` objects return ``{...}`` and :class:`list` objects + return ``[...]``. + + The function will return a negative integer if the recursion limit + is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should + typically return ``NULL``. + + Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` + implementation can continue normally. + +.. c:function:: void Py_ReprLeave(PyObject *object) + + Ends a :c:func:`Py_ReprEnter`. Must be called once for each + invocation of :c:func:`Py_ReprEnter` that returns zero. + + +.. _standardexceptions: + +Standard Exceptions +=================== + +All standard Python exceptions are available as global variables whose names are +``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + ++-----------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++=========================================+=================================+==========+ +| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NameError` | :exc:`NameError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | ++-----------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.3 + :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, + :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, + :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, + :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, + :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, + :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, + :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` + and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. + +.. versionadded:: 3.5 + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. + +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. + +These are compatibility aliases to :c:data:`PyExc_OSError`: + +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + ++-------------------------------------+----------+ +| C Name | Notes | ++=====================================+==========+ +| :c:data:`PyExc_EnvironmentError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_IOError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_WindowsError` | \(3) | ++-------------------------------------+----------+ + +.. versionchanged:: 3.3 + These aliases used to be separate exception types. + +Notes: + +(1) + This is a base class for other standard exceptions. + +(2) + Only defined on Windows; protect code that uses this by testing that the + preprocessor macro ``MS_WINDOWS`` is defined. + +.. _standardwarningcategories: + +Standard Warning Categories +=========================== + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DeprecationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/memory.rst.bak b/Doc/c-api/memory.rst.bak new file mode 100644 index 000000000000000..588668ee853c65a --- /dev/null +++ b/Doc/c-api/memory.rst.bak @@ -0,0 +1,642 @@ +.. highlight:: c + + +.. _memory: + +***************** +Memory Management +***************** + +.. sectionauthor:: Vladimir Marangozov + + + +.. _memoryoverview: + +Overview +======== + +Memory management in Python involves a private heap containing all Python +objects and data structures. The management of this private heap is ensured +internally by the *Python memory manager*. The Python memory manager has +different components which deal with various dynamic storage management aspects, +like sharing, segmentation, preallocation or caching. + +At the lowest level, a raw memory allocator ensures that there is enough room in +the private heap for storing all Python-related data by interacting with the +memory manager of the operating system. On top of the raw memory allocator, +several object-specific allocators operate on the same heap and implement +distinct memory management policies adapted to the peculiarities of every object +type. For example, integer objects are managed differently within the heap than +strings, tuples or dictionaries because integers imply different storage +requirements and speed/space tradeoffs. The Python memory manager thus delegates +some of the work to the object-specific allocators, but ensures that the latter +operate within the bounds of the private heap. + +It is important to understand that the management of the Python heap is +performed by the interpreter itself and that the user has no control over it, +even if they regularly manipulate object pointers to memory blocks inside that +heap. The allocation of heap space for Python objects and other internal +buffers is performed on demand by the Python memory manager through the Python/C +API functions listed in this document. + +.. index:: + single: malloc() + single: calloc() + single: realloc() + single: free() + +To avoid memory corruption, extension writers should never try to operate on +Python objects with the functions exported by the C library: :c:func:`malloc`, +:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed +calls between the C allocator and the Python memory manager with fatal +consequences, because they implement different algorithms and operate on +different heaps. However, one may safely allocate and release memory blocks +with the C library allocator for individual purposes, as shown in the following +example:: + + PyObject *res; + char *buf = (char *) malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + ...Do some I/O operation involving buf... + res = PyBytes_FromString(buf); + free(buf); /* malloc'ed */ + return res; + +In this example, the memory request for the I/O buffer is handled by the C +library allocator. The Python memory manager is involved only in the allocation +of the bytes object returned as a result. + +In most situations, however, it is recommended to allocate memory from the +Python heap specifically because the latter is under control of the Python +memory manager. For example, this is required when the interpreter is extended +with new object types written in C. Another reason for using the Python heap is +the desire to *inform* the Python memory manager about the memory needs of the +extension module. Even when the requested memory is used exclusively for +internal, highly-specific purposes, delegating all memory requests to the Python +memory manager causes the interpreter to have a more accurate image of its +memory footprint as a whole. Consequently, under certain circumstances, the +Python memory manager may or may not trigger appropriate actions, like garbage +collection, memory compaction or other preventive procedures. Note that by using +the C library allocator as shown in the previous example, the allocated memory +for the I/O buffer escapes completely the Python memory manager. + +.. seealso:: + + The :envvar:`PYTHONMALLOC` environment variable can be used to configure + the memory allocators used by Python. + + The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print + statistics of the :ref:`pymalloc memory allocator ` every time a + new pymalloc object arena is created, and on shutdown. + +Allocator Domains +================= + +All allocating functions belong to one of three different "domains" (see also +:c:type:`PyMemAllocatorDomain`). These domains represent different allocation +strategies and are optimized for different purposes. The specific details on +how every domain allocates memory or what internal functions each domain calls +is considered an implementation detail, but for debugging purposes a simplified +table can be found at :ref:`here `. There is no hard +requirement to use the memory returned by the allocation functions belonging to +a given domain for only the purposes hinted by that domain (although this is the +recommended practice). For example, one could use the memory returned by +:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned +by :c:func:`PyObject_Malloc` for allocating memory for buffers. + +The three allocation domains are: + +* Raw domain: intended for allocating memory for general-purpose memory + buffers where the allocation *must* go to the system allocator or where the + allocator can operate without the :term:`GIL`. The memory is requested directly + to the system. + +* "Mem" domain: intended for allocating memory for Python buffers and + general-purpose memory buffers where the allocation must be performed with + the :term:`GIL` held. The memory is taken from the Python private heap. + +* Object domain: intended for allocating memory belonging to Python objects. The + memory is taken from the Python private heap. + +When freeing memory previously allocated by the allocating functions belonging to a +given domain,the matching specific deallocating functions must be used. For example, +:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. + +Raw Memory Interface +==================== + +The following function sets are wrappers to the system allocator. These +functions are thread-safe, the :term:`GIL ` does not +need to be held. + +The :ref:`default raw memory allocator ` uses +the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` +and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting +zero bytes. + +.. versionadded:: 3.4 + +.. c:function:: void* PyMem_RawMalloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been + called instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will + be unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if + *n* is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or + :c:func:`PyMem_RawCalloc`. + + If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* + remains a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_RawFree(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or + :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been + called before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + + +.. _memoryinterface: + +Memory Interface +================ + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +The :ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. versionchanged:: 3.6 + + The default allocator is now pymalloc instead of system :c:func:`malloc`. + +.. c:function:: void* PyMem_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyMem_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyMem_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. + + If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or + :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + +The following type-oriented macros are provided for convenience. Note that +*TYPE* refers to any C type. + + +.. c:function:: TYPE* PyMem_New(TYPE, size_t n) + + Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of + memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have + been initialized in any way. + + +.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) + + Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, + *p* will be a pointer to the new memory area, or ``NULL`` in the event of + failure. + + This is a C preprocessor macro; *p* is always reassigned. Save the original + value of *p* to avoid losing memory when handling errors. + + +.. c:function:: void PyMem_Del(void *p) + + Same as :c:func:`PyMem_Free`. + +In addition, the following macro sets are provided for calling the Python memory +allocator directly, without involving the C API functions listed above. However, +note that their use does not preserve binary compatibility across Python +versions and is therefore deprecated in extension modules. + +* ``PyMem_MALLOC(size)`` +* ``PyMem_NEW(type, size)`` +* ``PyMem_REALLOC(ptr, size)`` +* ``PyMem_RESIZE(ptr, type, size)`` +* ``PyMem_FREE(ptr)`` +* ``PyMem_DEL(ptr)`` + + +Object allocators +================= + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +.. note:: + There is no guarantee that the memory returned by these allocators can be + succesfully casted to a Python object when intercepting the allocating + functions in this domain by the methods described in + the :ref:`Customize Memory Allocators ` section. + +The :ref:`default object allocator ` uses the +:ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. c:function:: void* PyObject_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyObject_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyObject_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. + + If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyObject_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or + :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + + +.. _default-memory-allocators: + +Default Memory Allocators +========================= + +Default memory allocators: + +=============================== ==================== ================== ===================== ==================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=============================== ==================== ================== ===================== ==================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +=============================== ==================== ================== ===================== ==================== + +Legend: + +* Name: value for :envvar:`PYTHONMALLOC` environment variable +* ``malloc``: system allocators from the standard C library, C functions: + :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` +* ``pymalloc``: :ref:`pymalloc memory allocator ` +* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` + +.. _customize-memory-allocators: + +Customize Memory Allocators +=========================== + +.. versionadded:: 3.4 + +.. c:type:: PyMemAllocatorEx + + Structure used to describe a memory block allocator. The structure has + four fields: + + +----------------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==========================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +----------------------------------------------------------+---------------------------------------+ + | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | + | | with zeros | + +----------------------------------------------------------+---------------------------------------+ + | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, void *ptr)`` | free a memory block | + +----------------------------------------------------------+---------------------------------------+ + + .. versionchanged:: 3.5 + The :c:type:`PyMemAllocator` structure was renamed to + :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. + + +.. c:type:: PyMemAllocatorDomain + + Enum used to identify an allocator domain. Domains: + + .. c:macro:: PYMEM_DOMAIN_RAW + + Functions: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + + .. c:macro:: PYMEM_DOMAIN_MEM + + Functions: + + * :c:func:`PyMem_Malloc`, + * :c:func:`PyMem_Realloc` + * :c:func:`PyMem_Calloc` + * :c:func:`PyMem_Free` + + .. c:macro:: PYMEM_DOMAIN_OBJ + + Functions: + + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:func:`PyObject_Free` + +.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) + + Get the memory block allocator of the specified domain. + + +.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) + + Set the memory block allocator of the specified domain. + + The new allocator must return a distinct non-``NULL`` pointer when requesting + zero bytes. + + For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be + thread-safe: the :term:`GIL ` is not held when the + allocator is called. + + If the new allocator is not a hook (does not call the previous allocator), + the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the + debug hooks on top on the new allocator. + + +.. c:function:: void PyMem_SetupDebugHooks(void) + + Setup hooks to detect bugs in the Python memory allocator functions. + + Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), + freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks + are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). + + Runtime checks: + + - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer + allocated by :c:func:`PyMem_Malloc` + - Detect write before the start of the buffer (buffer underflow) + - Detect write after the end of the buffer (buffer overflow) + - Check that the :term:`GIL ` is held when + allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + :c:func:`PyMem_Malloc`) domains are called + + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + + These hooks are :ref:`installed by default ` if + Python is compiled in debug + mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install + debug hooks on a Python compiled in release mode. + + .. versionchanged:: 3.6 + This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. The debug hooks now also check + if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + :c:data:`PYMEM_DOMAIN_MEM` domains are called. + + .. versionchanged:: 3.8 + Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and + ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` + and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` + and ``free()``. + + +.. _pymalloc: + +The pymalloc allocator +====================== + +Python has a *pymalloc* allocator optimized for small objects (smaller or equal +to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" +with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and +:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. + +*pymalloc* is the :ref:`default allocator ` of the +:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and +:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. + +The arena allocator uses the following functions: + +* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, +* :c:func:`mmap` and :c:func:`munmap` if available, +* :c:func:`malloc` and :c:func:`free` otherwise. + +Customize pymalloc Arena Allocator +---------------------------------- + +.. versionadded:: 3.4 + +.. c:type:: PyObjectArenaAllocator + + Structure used to describe an arena allocator. The structure has + three fields: + + +--------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +--------------------------------------------------+---------------------------------------+ + | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | + +--------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + +--------------------------------------------------+---------------------------------------+ + +.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) + + Get the arena allocator. + +.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) + + Set the arena allocator. + + +tracemalloc C API +================= + +.. versionadded:: 3.7 + +.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) + + Track an allocated memory block in the :mod:`tracemalloc` module. + + Return ``0`` on success, return ``-1`` on error (failed to allocate memory to + store the trace). Return ``-2`` if tracemalloc is disabled. + + If memory block is already tracked, update the existing trace. + +.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) + + Untrack an allocated memory block in the :mod:`tracemalloc` module. + Do nothing if the block was not tracked. + + Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. + + +.. _memoryexamples: + +Examples +======== + +Here is the example from section :ref:`memoryoverview`, rewritten so that the +I/O buffer is allocated from the Python heap by using the first function set:: + + PyObject *res; + char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyBytes_FromString(buf); + PyMem_Free(buf); /* allocated with PyMem_Malloc */ + return res; + +The same code using the type-oriented function set:: + + PyObject *res; + char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyBytes_FromString(buf); + PyMem_Del(buf); /* allocated with PyMem_New */ + return res; + +Note that in the two examples above, the buffer is always manipulated via +functions belonging to the same set. Indeed, it is required to use the same +memory API family for a given memory block, so that the risk of mixing different +allocators is reduced to a minimum. The following code sequence contains two +errors, one of which is labeled as *fatal* because it mixes two different +allocators operating on different heaps. :: + + char *buf1 = PyMem_New(char, BUFSIZ); + char *buf2 = (char *) malloc(BUFSIZ); + char *buf3 = (char *) PyMem_Malloc(BUFSIZ); + ... + PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ + free(buf2); /* Right -- allocated via malloc() */ + free(buf1); /* Fatal -- should be PyMem_Del() */ + +In addition to the functions aimed at handling raw memory blocks from the Python +heap, objects in Python are allocated and released with :c:func:`PyObject_New`, +:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. + +These will be explained in the next chapter on defining and implementing new +object types in C. diff --git a/Doc/faq/design.rst.bak b/Doc/faq/design.rst.bak new file mode 100644 index 000000000000000..7fe1c6d58f58a12 --- /dev/null +++ b/Doc/faq/design.rst.bak @@ -0,0 +1,759 @@ +====================== +Design and History FAQ +====================== + +.. only:: html + + .. contents:: + + +Why does Python use indentation for grouping of statements? +----------------------------------------------------------- + +Guido van Rossum believes that using indentation for grouping is extremely +elegant and contributes a lot to the clarity of the average Python program. +Most people learn to love this feature after a while. + +Since there are no begin/end brackets there cannot be a disagreement between +grouping perceived by the parser and the human reader. Occasionally C +programmers will encounter a fragment of code like this:: + + if (x <= y) + x++; + y--; + z++; + +Only the ``x++`` statement is executed if the condition is true, but the +indentation leads many to believe otherwise. Even experienced C programmers will +sometimes stare at it a long time wondering as to why ``y`` is being decremented even +for ``x > y``. + +Because there are no begin/end brackets, Python is much less prone to +coding-style conflicts. In C there are many different ways to place the braces. +After becoming used to reading and writing code using a particular style, +it is normal to feel somewhat uneasy when reading (or being required to write) +in a different one. + + +Many coding styles place begin/end brackets on a line by themselves. This makes +programs considerably longer and wastes valuable screen space, making it harder +to get a good overview of a program. Ideally, a function should fit on one +screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 +lines of C. This is not solely due to the lack of begin/end brackets -- the +lack of declarations and the high-level data types are also responsible -- but +the indentation-based syntax certainly helps. + + +Why am I getting strange results with simple arithmetic operations? +------------------------------------------------------------------- + +See the next question. + + +Why are floating-point calculations so inaccurate? +-------------------------------------------------- + +Users are often surprised by results like this:: + + >>> 1.2 - 1.0 + 0.19999999999999996 + +and think it is a bug in Python. It's not. This has little to do with Python, +and much more to do with how the underlying platform handles floating-point +numbers. + +The :class:`float` type in CPython uses a C ``double`` for storage. A +:class:`float` object's value is stored in binary floating-point with a fixed +precision (typically 53 bits) and Python uses C operations, which in turn rely +on the hardware implementation in the processor, to perform floating-point +operations. This means that as far as floating-point operations are concerned, +Python behaves like many popular languages including C and Java. + +Many numbers that can be written easily in decimal notation cannot be expressed +exactly in binary floating-point. For example, after:: + + >>> x = 1.2 + +the value stored for ``x`` is a (very good) approximation to the decimal value +``1.2``, but is not exactly equal to it. On a typical machine, the actual +stored value is:: + + 1.0011001100110011001100110011001100110011001100110011 (binary) + +which is exactly:: + + 1.1999999999999999555910790149937383830547332763671875 (decimal) + +The typical precision of 53 bits provides Python floats with 15--16 +decimal digits of accuracy. + +For a fuller explanation, please see the :ref:`floating point arithmetic +` chapter in the Python tutorial. + + +Why are Python strings immutable? +--------------------------------- + +There are several advantages. + +One is performance: knowing that a string is immutable means we can allocate +space for it at creation time, and the storage requirements are fixed and +unchanging. This is also one of the reasons for the distinction between tuples +and lists. + +Another advantage is that strings in Python are considered as "elemental" as +numbers. No amount of activity will change the value 8 to anything else, and in +Python, no amount of activity will change the string "eight" to anything else. + + +.. _why-self: + +Why must 'self' be used explicitly in method definitions and calls? +------------------------------------------------------------------- + +The idea was borrowed from Modula-3. It turns out to be very useful, for a +variety of reasons. + +First, it's more obvious that you are using a method or instance attribute +instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it +absolutely clear that an instance variable or method is used even if you don't +know the class definition by heart. In C++, you can sort of tell by the lack of +a local variable declaration (assuming globals are rare or easily recognizable) +-- but in Python, there are no local variable declarations, so you'd have to +look up the class definition to be sure. Some C++ and Java coding standards +call for instance attributes to have an ``m_`` prefix, so this explicitness is +still useful in those languages, too. + +Second, it means that no special syntax is necessary if you want to explicitly +reference or call the method from a particular class. In C++, if you want to +use a method from a base class which is overridden in a derived class, you have +to use the ``::`` operator -- in Python you can write +``baseclass.methodname(self, )``. This is particularly useful +for :meth:`__init__` methods, and in general in cases where a derived class +method wants to extend the base class method of the same name and thus has to +call the base class method somehow. + +Finally, for instance variables it solves a syntactic problem with assignment: +since local variables in Python are (by definition!) those variables to which a +value is assigned in a function body (and that aren't explicitly declared +global), there has to be some way to tell the interpreter that an assignment was +meant to assign to an instance variable instead of to a local variable, and it +should preferably be syntactic (for efficiency reasons). C++ does this through +declarations, but Python doesn't have declarations and it would be a pity having +to introduce them just for this purpose. Using the explicit ``self.var`` solves +this nicely. Similarly, for using instance variables, having to write +``self.var`` means that references to unqualified names inside a method don't +have to search the instance's directories. To put it another way, local +variables and instance variables live in two different namespaces, and you need +to tell Python which namespace to use. + + +.. _why-can-t-i-use-an-assignment-in-an-expression: + +Why can't I use an assignment in an expression? +----------------------------------------------- + +Starting in Python 3.8, you can! + +Assignment expressions using the walrus operator `:=` assign a variable in an +expression:: + + while chunk := fp.read(200): + print(chunk) + +See :pep:`572` for more information. + + + +Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? +---------------------------------------------------------------------------------------------------------------- + +As Guido said: + + (a) For some operations, prefix notation just reads better than + postfix -- prefix (and infix!) operations have a long tradition in + mathematics which likes notations where the visuals help the + mathematician thinking about a problem. Compare the easy with which we + rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of + doing the same thing using a raw OO notation. + + (b) When I read code that says len(x) I *know* that it is asking for + the length of something. This tells me two things: the result is an + integer, and the argument is some kind of container. To the contrary, + when I read x.len(), I have to already know that x is some kind of + container implementing an interface or inheriting from a class that + has a standard len(). Witness the confusion we occasionally have when + a class that is not implementing a mapping has a get() or keys() + method, or something that isn't a file has a write() method. + + -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html + + +Why is join() a string method instead of a list or tuple method? +---------------------------------------------------------------- + +Strings became much more like other standard types starting in Python 1.6, when +methods were added which give the same functionality that has always been +available using the functions of the string module. Most of these new methods +have been widely accepted, but the one which appears to make some programmers +feel uncomfortable is:: + + ", ".join(['1', '2', '4', '8', '16']) + +which gives the result:: + + "1, 2, 4, 8, 16" + +There are two common arguments against this usage. + +The first runs along the lines of: "It looks really ugly using a method of a +string literal (string constant)", to which the answer is that it might, but a +string literal is just a fixed value. If the methods are to be allowed on names +bound to strings there is no logical reason to make them unavailable on +literals. + +The second objection is typically cast as: "I am really telling a sequence to +join its members together with a string constant". Sadly, you aren't. For some +reason there seems to be much less difficulty with having :meth:`~str.split` as +a string method, since in that case it is easy to see that :: + + "1, 2, 4, 8, 16".split(", ") + +is an instruction to a string literal to return the substrings delimited by the +given separator (or, by default, arbitrary runs of white space). + +:meth:`~str.join` is a string method because in using it you are telling the +separator string to iterate over a sequence of strings and insert itself between +adjacent elements. This method can be used with any argument which obeys the +rules for sequence objects, including any new classes you might define yourself. +Similar methods exist for bytes and bytearray objects. + + +How fast are exceptions? +------------------------ + +A try/except block is extremely efficient if no exceptions are raised. Actually +catching an exception is expensive. In versions of Python prior to 2.0 it was +common to use this idiom:: + + try: + value = mydict[key] + except KeyError: + mydict[key] = getvalue(key) + value = mydict[key] + +This only made sense when you expected the dict to have the key almost all the +time. If that wasn't the case, you coded it like this:: + + if key in mydict: + value = mydict[key] + else: + value = mydict[key] = getvalue(key) + +For this specific case, you could also use ``value = dict.setdefault(key, +getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it +is evaluated in all cases. + + +Why isn't there a switch or case statement in Python? +----------------------------------------------------- + +You can do this easily enough with a sequence of ``if... elif... elif... else``. +For literal values, or constants within a namespace, you can also use a +``match ... case`` statement. + +For cases where you need to choose from a very large number of possibilities, +you can create a dictionary mapping case values to functions to call. For +example:: + + def function_1(...): + ... + + functions = {'a': function_1, + 'b': function_2, + 'c': self.method_1, ...} + + func = functions[value] + func() + +For calling methods on objects, you can simplify yet further by using the +:func:`getattr` built-in to retrieve methods with a particular name:: + + def visit_a(self, ...): + ... + ... + + def dispatch(self, value): + method_name = 'visit_' + str(value) + method = getattr(self, method_name) + method() + +It's suggested that you use a prefix for the method names, such as ``visit_`` in +this example. Without such a prefix, if values are coming from an untrusted +source, an attacker would be able to call any method on your object. + + +Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? +-------------------------------------------------------------------------------------------------------- + +Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for +each Python stack frame. Also, extensions can call back into Python at almost +random moments. Therefore, a complete threads implementation requires thread +support for C. + +Answer 2: Fortunately, there is `Stackless Python `_, +which has a completely redesigned interpreter loop that avoids the C stack. + + +Why can't lambda expressions contain statements? +------------------------------------------------ + +Python lambda expressions cannot contain statements because Python's syntactic +framework can't handle statements nested inside expressions. However, in +Python, this is not a serious problem. Unlike lambda forms in other languages, +where they add functionality, Python lambdas are only a shorthand notation if +you're too lazy to define a function. + +Functions are already first class objects in Python, and can be declared in a +local scope. Therefore the only advantage of using a lambda instead of a +locally-defined function is that you don't need to invent a name for the +function -- but that's just a local variable to which the function object (which +is exactly the same type of object that a lambda expression yields) is assigned! + + +Can Python be compiled to machine code, C or some other language? +----------------------------------------------------------------- + +`Cython `_ compiles a modified version of Python with +optional annotations into C extensions. `Nuitka `_ is +an up-and-coming compiler of Python into C++ code, aiming to support the full +Python language. For compiling to Java you can consider +`VOC `_. + + +How does Python manage memory? +------------------------------ + +The details of Python memory management depend on the implementation. The +standard implementation of Python, :term:`CPython`, uses reference counting to +detect inaccessible objects, and another mechanism to collect reference cycles, +periodically executing a cycle detection algorithm which looks for inaccessible +cycles and deletes the objects involved. The :mod:`gc` module provides functions +to perform a garbage collection, obtain debugging statistics, and tune the +collector's parameters. + +Other implementations (such as `Jython `_ or +`PyPy `_), however, can rely on a different mechanism +such as a full-blown garbage collector. This difference can cause some +subtle porting problems if your Python code depends on the behavior of the +reference counting implementation. + +In some Python implementations, the following code (which is fine in CPython) +will probably run out of file descriptors:: + + for file in very_long_list_of_files: + f = open(file) + c = f.read(1) + +Indeed, using CPython's reference counting and destructor scheme, each new +assignment to *f* closes the previous file. With a traditional GC, however, +those file objects will only get collected (and closed) at varying and possibly +long intervals. + +If you want to write code that will work with any Python implementation, +you should explicitly close the file or use the :keyword:`with` statement; +this will work regardless of memory management scheme:: + + for file in very_long_list_of_files: + with open(file) as f: + c = f.read(1) + + +Why doesn't CPython use a more traditional garbage collection scheme? +--------------------------------------------------------------------- + +For one thing, this is not a C standard feature and hence it's not portable. +(Yes, we know about the Boehm GC library. It has bits of assembler code for +*most* common platforms, not for all of them, and although it is mostly +transparent, it isn't completely transparent; patches are required to get +Python to work with it.) + +Traditional GC also becomes a problem when Python is embedded into other +applications. While in a standalone Python it's fine to replace the standard +malloc() and free() with versions provided by the GC library, an application +embedding Python may want to have its *own* substitute for malloc() and free(), +and may not want Python's. Right now, CPython works with anything that +implements malloc() and free() properly. + + +Why isn't all memory freed when CPython exits? +---------------------------------------------- + +Objects referenced from the global namespaces of Python modules are not always +deallocated when Python exits. This may happen if there are circular +references. There are also certain bits of memory that are allocated by the C +library that are impossible to free (e.g. a tool like Purify will complain about +these). Python is, however, aggressive about cleaning up memory on exit and +does try to destroy every single object. + +If you want to force Python to delete certain things on deallocation use the +:mod:`atexit` module to run a function that will force those deletions. + + +Why are there separate tuple and list data types? +------------------------------------------------- + +Lists and tuples, while similar in many respects, are generally used in +fundamentally different ways. Tuples can be thought of as being similar to +Pascal records or C structs; they're small collections of related data which may +be of different types which are operated on as a group. For example, a +Cartesian coordinate is appropriately represented as a tuple of two or three +numbers. + +Lists, on the other hand, are more like arrays in other languages. They tend to +hold a varying number of objects all of which have the same type and which are +operated on one-by-one. For example, ``os.listdir('.')`` returns a list of +strings representing the files in the current directory. Functions which +operate on this output would generally not break if you added another file or +two to the directory. + +Tuples are immutable, meaning that once a tuple has been created, you can't +replace any of its elements with a new value. Lists are mutable, meaning that +you can always change a list's elements. Only immutable elements can be used as +dictionary keys, and hence only tuples and not lists can be used as keys. + + +How are lists implemented in CPython? +------------------------------------- + +CPython's lists are really variable-length arrays, not Lisp-style linked lists. +The implementation uses a contiguous array of references to other objects, and +keeps a pointer to this array and the array's length in a list head structure. + +This makes indexing a list ``a[i]`` an operation whose cost is independent of +the size of the list or the value of the index. + +When items are appended or inserted, the array of references is resized. Some +cleverness is applied to improve the performance of appending items repeatedly; +when the array must be grown, some extra space is allocated so the next few +times don't require an actual resize. + + +How are dictionaries implemented in CPython? +-------------------------------------------- + +CPython's dictionaries are implemented as resizable hash tables. Compared to +B-trees, this gives better performance for lookup (the most common operation by +far) under most circumstances, and the implementation is simpler. + +Dictionaries work by computing a hash code for each key stored in the dictionary +using the :func:`hash` built-in function. The hash code varies widely depending +on the key and a per-process seed; for example, "Python" could hash to +-539294296 while "python", a string that differs by a single bit, could hash +to 1142331976. The hash code is then used to calculate a location in an +internal array where the value will be stored. Assuming that you're storing +keys that all have different hash values, this means that dictionaries take +constant time -- O(1), in Big-O notation -- to retrieve a key. + + +Why must dictionary keys be immutable? +-------------------------------------- + +The hash table implementation of dictionaries uses a hash value calculated from +the key value to find the key. If the key were a mutable object, its value +could change, and thus its hash could also change. But since whoever changes +the key object can't tell that it was being used as a dictionary key, it can't +move the entry around in the dictionary. Then, when you try to look up the same +object in the dictionary it won't be found because its hash value is different. +If you tried to look up the old value it wouldn't be found either, because the +value of the object found in that hash bin would be different. + +If you want a dictionary indexed with a list, simply convert the list to a tuple +first; the function ``tuple(L)`` creates a tuple with the same entries as the +list ``L``. Tuples are immutable and can therefore be used as dictionary keys. + +Some unacceptable solutions that have been proposed: + +- Hash lists by their address (object ID). This doesn't work because if you + construct a new list with the same value it won't be found; e.g.:: + + mydict = {[1, 2]: '12'} + print(mydict[[1, 2]]) + + would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the + second line differs from that in the first line. In other words, dictionary + keys should be compared using ``==``, not using :keyword:`is`. + +- Make a copy when using a list as a key. This doesn't work because the list, + being a mutable object, could contain a reference to itself, and then the + copying code would run into an infinite loop. + +- Allow lists as keys but tell the user not to modify them. This would allow a + class of hard-to-track bugs in programs when you forgot or modified a list by + accident. It also invalidates an important invariant of dictionaries: every + value in ``d.keys()`` is usable as a key of the dictionary. + +- Mark lists as read-only once they are used as a dictionary key. The problem + is that it's not just the top-level object that could change its value; you + could use a tuple containing a list as a key. Entering anything as a key into + a dictionary would require marking all objects reachable from there as + read-only -- and again, self-referential objects could cause an infinite loop. + +There is a trick to get around this if you need to, but use it at your own risk: +You can wrap a mutable structure inside a class instance which has both a +:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the +hash value for all such wrapper objects that reside in a dictionary (or other +hash based structure), remain fixed while the object is in the dictionary (or +other structure). :: + + class ListWrapper: + def __init__(self, the_list): + self.the_list = the_list + + def __eq__(self, other): + return self.the_list == other.the_list + + def __hash__(self): + l = self.the_list + result = 98767 - len(l)*555 + for i, el in enumerate(l): + try: + result = result + (hash(el) % 9999999) * 1001 + i + except Exception: + result = (result % 7777777) + i * 333 + return result + +Note that the hash computation is complicated by the possibility that some +members of the list may be unhashable and also by the possibility of arithmetic +overflow. + +Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) +is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), +regardless of whether the object is in a dictionary or not. If you fail to meet +these restrictions dictionaries and other hash based structures will misbehave. + +In the case of ListWrapper, whenever the wrapper object is in a dictionary the +wrapped list must not change to avoid anomalies. Don't do this unless you are +prepared to think hard about the requirements and the consequences of not +meeting them correctly. Consider yourself warned. + + +Why doesn't list.sort() return the sorted list? +----------------------------------------------- + +In situations where performance matters, making a copy of the list just to sort +it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In +order to remind you of that fact, it does not return the sorted list. This way, +you won't be fooled into accidentally overwriting a list when you need a sorted +copy but also need to keep the unsorted version around. + +If you want to return a new list, use the built-in :func:`sorted` function +instead. This function creates a new list from a provided iterable, sorts +it and returns it. For example, here's how to iterate over the keys of a +dictionary in sorted order:: + + for key in sorted(mydict): + ... # do whatever with mydict[key]... + + +How do you specify and enforce an interface spec in Python? +----------------------------------------------------------- + +An interface specification for a module as provided by languages such as C++ and +Java describes the prototypes for the methods and functions of the module. Many +feel that compile-time enforcement of interface specifications helps in the +construction of large programs. + +Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes +(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check +whether an instance or a class implements a particular ABC. The +:mod:`collections.abc` module defines a set of useful ABCs such as +:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and +:class:`~collections.abc.MutableMapping`. + +For Python, many of the advantages of interface specifications can be obtained +by an appropriate test discipline for components. + +A good test suite for a module can both provide a regression test and serve as a +module interface specification and a set of examples. Many Python modules can +be run as a script to provide a simple "self test." Even modules which use +complex external interfaces can often be tested in isolation using trivial +"stub" emulations of the external interface. The :mod:`doctest` and +:mod:`unittest` modules or third-party test frameworks can be used to construct +exhaustive test suites that exercise every line of code in a module. + +An appropriate testing discipline can help build large complex applications in +Python as well as having interface specifications would. In fact, it can be +better because an interface specification cannot test certain properties of a +program. For example, the :meth:`append` method is expected to add new elements +to the end of some internal list; an interface specification cannot test that +your :meth:`append` implementation will actually do this correctly, but it's +trivial to check this property in a test suite. + +Writing test suites is very helpful, and you might want to design your code to +make it easily tested. One increasingly popular technique, test-driven +development, calls for writing parts of the test suite first, before you write +any of the actual code. Of course Python allows you to be sloppy and not write +test cases at all. + + +Why is there no goto? +--------------------- + +You can use exceptions to provide a "structured goto" that even works across +function calls. Many feel that exceptions can conveniently emulate all +reasonable uses of the "go" or "goto" constructs of C, Fortran, and other +languages. For example:: + + class label(Exception): pass # declare a label + + try: + ... + if condition: raise label() # goto label + ... + except label: # where to goto + pass + ... + +This doesn't allow you to jump into the middle of a loop, but that's usually +considered an abuse of goto anyway. Use sparingly. + + +Why can't raw strings (r-strings) end with a backslash? +------------------------------------------------------- + +More precisely, they can't end with an odd number of backslashes: the unpaired +backslash at the end escapes the closing quote character, leaving an +unterminated string. + +Raw strings were designed to ease creating input for processors (chiefly regular +expression engines) that want to do their own backslash escape processing. Such +processors consider an unmatched trailing backslash to be an error anyway, so +raw strings disallow that. In return, they allow you to pass on the string +quote character by escaping it with a backslash. These rules work well when +r-strings are used for their intended purpose. + +If you're trying to build Windows pathnames, note that all Windows system calls +accept forward slashes too:: + + f = open("/mydir/file.txt") # works fine! + +If you're trying to build a pathname for a DOS command, try e.g. one of :: + + dir = r"\this\is\my\dos\dir" "\\" + dir = r"\this\is\my\dos\dir\ "[:-1] + dir = "\\this\\is\\my\\dos\\dir\\" + + +Why doesn't Python have a "with" statement for attribute assignments? +--------------------------------------------------------------------- + +Python has a 'with' statement that wraps the execution of a block, calling code +on the entrance and exit from the block. Some languages have a construct that +looks like this:: + + with obj: + a = 1 # equivalent to obj.a = 1 + total = total + 1 # obj.total = obj.total + 1 + +In Python, such a construct would be ambiguous. + +Other languages, such as Object Pascal, Delphi, and C++, use static types, so +it's possible to know, in an unambiguous way, what member is being assigned +to. This is the main point of static typing -- the compiler *always* knows the +scope of every variable at compile time. + +Python uses dynamic types. It is impossible to know in advance which attribute +will be referenced at runtime. Member attributes may be added or removed from +objects on the fly. This makes it impossible to know, from a simple reading, +what attribute is being referenced: a local one, a global one, or a member +attribute? + +For instance, take the following incomplete snippet:: + + def foo(a): + with a: + print(x) + +The snippet assumes that "a" must have a member attribute called "x". However, +there is nothing in Python that tells the interpreter this. What should happen +if "a" is, let us say, an integer? If there is a global variable named "x", +will it be used inside the with block? As you see, the dynamic nature of Python +makes such choices much harder. + +The primary benefit of "with" and similar language features (reduction of code +volume) can, however, easily be achieved in Python by assignment. Instead of:: + + function(args).mydict[index][index].a = 21 + function(args).mydict[index][index].b = 42 + function(args).mydict[index][index].c = 63 + +write this:: + + ref = function(args).mydict[index][index] + ref.a = 21 + ref.b = 42 + ref.c = 63 + +This also has the side-effect of increasing execution speed because name +bindings are resolved at run-time in Python, and the second version only needs +to perform the resolution once. + + +Why are colons required for the if/while/def/class statements? +-------------------------------------------------------------- + +The colon is required primarily to enhance readability (one of the results of +the experimental ABC language). Consider this:: + + if a == b + print(a) + +versus :: + + if a == b: + print(a) + +Notice how the second one is slightly easier to read. Notice further how a +colon sets off the example in this FAQ answer; it's a standard usage in English. + +Another minor reason is that the colon makes it easier for editors with syntax +highlighting; they can look for colons to decide when indentation needs to be +increased instead of having to do a more elaborate parsing of the program text. + + +Why does Python allow commas at the end of lists and tuples? +------------------------------------------------------------ + +Python lets you add a trailing comma at the end of lists, tuples, and +dictionaries:: + + [1, 2, 3,] + ('a', 'b', 'c',) + d = { + "A": [1, 5], + "B": [6, 7], # last trailing comma is optional but good style + } + + +There are several reasons to allow this. + +When you have a literal value for a list, tuple, or dictionary spread across +multiple lines, it's easier to add more elements because you don't have to +remember to add a comma to the previous line. The lines can also be reordered +without creating a syntax error. + +Accidentally omitting the comma can lead to errors that are hard to diagnose. +For example:: + + x = [ + "fee", + "fie" + "foo", + "fum" + ] + +This list looks like it has four elements, but it actually contains three: +"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. + +Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/howto/descriptor.rst.bak b/Doc/howto/descriptor.rst.bak new file mode 100644 index 000000000000000..94a8b4e6b40b962 --- /dev/null +++ b/Doc/howto/descriptor.rst.bak @@ -0,0 +1,1575 @@ +.. _descriptorhowto: + +====================== +Descriptor HowTo Guide +====================== + +:Author: Raymond Hettinger +:Contact: + +.. Contents:: + + +:term:`Descriptors ` let objects customize attribute lookup, +storage, and deletion. + +This guide has four major sections: + +1) The "primer" gives a basic overview, moving gently from simple examples, + adding one feature at a time. Start here if you're new to descriptors. + +2) The second section shows a complete, practical descriptor example. If you + already know the basics, start there. + +3) The third section provides a more technical tutorial that goes into the + detailed mechanics of how descriptors work. Most people don't need this + level of detail. + +4) The last section has pure Python equivalents for built-in descriptors that + are written in C. Read this if you're curious about how functions turn + into bound methods or about the implementation of common tools like + :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :term:`__slots__`. + + +Primer +^^^^^^ + +In this primer, we start with the most basic possible example and then we'll +add new capabilities one by one. + + +Simple example: A descriptor that returns a constant +---------------------------------------------------- + +The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +returns the constant ``10``: + +.. testcode:: + + class Ten: + def __get__(self, obj, objtype=None): + return 10 + +To use the descriptor, it must be stored as a class variable in another class: + +.. testcode:: + + class A: + x = 5 # Regular class attribute + y = Ten() # Descriptor instance + +An interactive session shows the difference between normal attribute lookup +and descriptor lookup: + +.. doctest:: + + >>> a = A() # Make an instance of class A + >>> a.x # Normal attribute lookup + 5 + >>> a.y # Descriptor lookup + 10 + +In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` +in the class dictionary. In the ``a.y`` lookup, the dot operator +finds a descriptor instance, recognized by its ``__get__`` method. +Calling that method returns ``10``. + +Note that the value ``10`` is not stored in either the class dictionary or the +instance dictionary. Instead, the value ``10`` is computed on demand. + +This example shows how a simple descriptor works, but it isn't very useful. +For retrieving constants, normal attribute lookup would be better. + +In the next section, we'll create something more useful, a dynamic lookup. + + +Dynamic lookups +--------------- + +Interesting descriptors typically run computations instead of returning +constants: + +.. testcode:: + + import os + + class DirectorySize: + + def __get__(self, obj, objtype=None): + return len(os.listdir(obj.dirname)) + + class Directory: + + size = DirectorySize() # Descriptor instance + + def __init__(self, dirname): + self.dirname = dirname # Regular instance attribute + +An interactive session shows that the lookup is dynamic — it computes +different, updated answers each time:: + + >>> s = Directory('songs') + >>> g = Directory('games') + >>> s.size # The songs directory has twenty files + 20 + >>> g.size # The games directory has three files + 3 + >>> open('games/newfile').close() # Add a fourth file to the directory + >>> g.size # File count is automatically updated + 4 + +Besides showing how descriptors can run computations, this example also +reveals the purpose of the parameters to :meth:`__get__`. The *self* +parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is +either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that +lets the :meth:`__get__` method learn the target directory. The *objtype* +parameter is the class *Directory*. + + +Managed attributes +------------------ + +A popular use for descriptors is managing access to instance data. The +descriptor is assigned to a public attribute in the class dictionary while the +actual data is stored as a private attribute in the instance dictionary. The +descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +the public attribute is accessed. + +In the following example, *age* is the public attribute and *_age* is the +private attribute. When the public attribute is accessed, the descriptor logs +the lookup or update: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAgeAccess: + + def __get__(self, obj, objtype=None): + value = obj._age + logging.info('Accessing %r giving %r', 'age', value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', 'age', value) + obj._age = value + + class Person: + + age = LoggedAgeAccess() # Descriptor instance + + def __init__(self, name, age): + self.name = name # Regular instance attribute + self.age = age # Calls __set__() + + def birthday(self): + self.age += 1 # Calls both __get__() and __set__() + + +An interactive session shows that all access to the managed attribute *age* is +logged, but that the regular attribute *name* is not logged: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> mary = Person('Mary M', 30) # The initial age update is logged + INFO:root:Updating 'age' to 30 + >>> dave = Person('David D', 40) + INFO:root:Updating 'age' to 40 + + >>> vars(mary) # The actual data is in a private attribute + {'name': 'Mary M', '_age': 30} + >>> vars(dave) + {'name': 'David D', '_age': 40} + + >>> mary.age # Access the data and log the lookup + INFO:root:Accessing 'age' giving 30 + 30 + >>> mary.birthday() # Updates are logged as well + INFO:root:Accessing 'age' giving 30 + INFO:root:Updating 'age' to 31 + + >>> dave.name # Regular attribute lookup isn't logged + 'David D' + >>> dave.age # Only the managed attribute is logged + INFO:root:Accessing 'age' giving 40 + 40 + +One major issue with this example is that the private name *_age* is hardwired in +the *LoggedAgeAccess* class. That means that each instance can only have one +logged attribute and that its name is unchangeable. In the next example, +we'll fix that problem. + + +Customized names +---------------- + +When a class uses descriptors, it can inform each descriptor about which +variable name was used. + +In this example, the :class:`Person` class has two descriptor instances, +*name* and *age*. When the :class:`Person` class is defined, it makes a +callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +be recorded, giving each descriptor its own *public_name* and *private_name*: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAccess: + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + value = getattr(obj, self.private_name) + logging.info('Accessing %r giving %r', self.public_name, value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', self.public_name, value) + setattr(obj, self.private_name, value) + + class Person: + + name = LoggedAccess() # First descriptor instance + age = LoggedAccess() # Second descriptor instance + + def __init__(self, name, age): + self.name = name # Calls the first descriptor + self.age = age # Calls the second descriptor + + def birthday(self): + self.age += 1 + +An interactive session shows that the :class:`Person` class has called +:meth:`__set_name__` so that the field names would be recorded. Here +we call :func:`vars` to look up the descriptor without triggering it: + +.. doctest:: + + >>> vars(vars(Person)['name']) + {'public_name': 'name', 'private_name': '_name'} + >>> vars(vars(Person)['age']) + {'public_name': 'age', 'private_name': '_age'} + +The new class now logs access to both *name* and *age*: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> pete = Person('Peter P', 10) + INFO:root:Updating 'name' to 'Peter P' + INFO:root:Updating 'age' to 10 + >>> kate = Person('Catherine C', 20) + INFO:root:Updating 'name' to 'Catherine C' + INFO:root:Updating 'age' to 20 + +The two *Person* instances contain only the private names:: + + >>> vars(pete) + {'_name': 'Peter P', '_age': 10} + >>> vars(kate) + {'_name': 'Catherine C', '_age': 20} + + +Closing thoughts +---------------- + +A :term:`descriptor` is what we call any object that defines :meth:`__get__`, +:meth:`__set__`, or :meth:`__delete__`. + +Optionally, descriptors can have a :meth:`__set_name__` method. This is only +used in cases where a descriptor needs to know either the class where it was +created or the name of class variable it was assigned to. (This method, if +present, is called even if the class is not a descriptor.) + +Descriptors get invoked by the dot operator during attribute lookup. If a +descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, +the descriptor instance is returned without invoking it. + +Descriptors only work when used as class variables. When put in instances, +they have no effect. + +The main motivation for descriptors is to provide a hook allowing objects +stored in class variables to control what happens during attribute lookup. + +Traditionally, the calling class controls what happens during lookup. +Descriptors invert that relationship and allow the data being looked-up to +have a say in the matter. + +Descriptors are used throughout the language. It is how functions turn into +bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, +:func:`property`, and :func:`functools.cached_property` are all implemented as +descriptors. + + +Complete Practical Example +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this example, we create a practical and powerful tool for locating +notoriously hard to find data corruption bugs. + + +Validator class +--------------- + +A validator is a descriptor for managed attribute access. Prior to storing +any data, it verifies that the new value meets various type and range +restrictions. If those restrictions aren't met, it raises an exception to +prevent data corruption at its source. + +This :class:`Validator` class is both an :term:`abstract base class` and a +managed attribute descriptor: + +.. testcode:: + + from abc import ABC, abstractmethod + + class Validator(ABC): + + def __set_name__(self, owner, name): + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + return getattr(obj, self.private_name) + + def __set__(self, obj, value): + self.validate(value) + setattr(obj, self.private_name, value) + + @abstractmethod + def validate(self, value): + pass + +Custom validators need to inherit from :class:`Validator` and must supply a +:meth:`validate` method to test various restrictions as needed. + + +Custom validators +----------------- + +Here are three practical data validation utilities: + +1) :class:`OneOf` verifies that a value is one of a restricted set of options. + +2) :class:`Number` verifies that a value is either an :class:`int` or + :class:`float`. Optionally, it verifies that a value is between a given + minimum or maximum. + +3) :class:`String` verifies that a value is a :class:`str`. Optionally, it + validates a given minimum or maximum length. It can validate a + user-defined `predicate + `_ as well. + +.. testcode:: + + class OneOf(Validator): + + def __init__(self, *options): + self.options = set(options) + + def validate(self, value): + if value not in self.options: + raise ValueError(f'Expected {value!r} to be one of {self.options!r}') + + class Number(Validator): + + def __init__(self, minvalue=None, maxvalue=None): + self.minvalue = minvalue + self.maxvalue = maxvalue + + def validate(self, value): + if not isinstance(value, (int, float)): + raise TypeError(f'Expected {value!r} to be an int or float') + if self.minvalue is not None and value < self.minvalue: + raise ValueError( + f'Expected {value!r} to be at least {self.minvalue!r}' + ) + if self.maxvalue is not None and value > self.maxvalue: + raise ValueError( + f'Expected {value!r} to be no more than {self.maxvalue!r}' + ) + + class String(Validator): + + def __init__(self, minsize=None, maxsize=None, predicate=None): + self.minsize = minsize + self.maxsize = maxsize + self.predicate = predicate + + def validate(self, value): + if not isinstance(value, str): + raise TypeError(f'Expected {value!r} to be an str') + if self.minsize is not None and len(value) < self.minsize: + raise ValueError( + f'Expected {value!r} to be no smaller than {self.minsize!r}' + ) + if self.maxsize is not None and len(value) > self.maxsize: + raise ValueError( + f'Expected {value!r} to be no bigger than {self.maxsize!r}' + ) + if self.predicate is not None and not self.predicate(value): + raise ValueError( + f'Expected {self.predicate} to be true for {value!r}' + ) + + +Practical application +--------------------- + +Here's how the data validators can be used in a real class: + +.. testcode:: + + class Component: + + name = String(minsize=3, maxsize=10, predicate=str.isupper) + kind = OneOf('wood', 'metal', 'plastic') + quantity = Number(minvalue=0) + + def __init__(self, name, kind, quantity): + self.name = name + self.kind = kind + self.quantity = quantity + +The descriptors prevent invalid instances from being created: + +.. doctest:: + + >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase + Traceback (most recent call last): + ... + ValueError: Expected to be true for 'Widget' + + >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled + Traceback (most recent call last): + ... + ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} + + >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative + Traceback (most recent call last): + ... + ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number + Traceback (most recent call last): + ... + TypeError: Expected 'V' to be an int or float + + >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid + + +Technical Tutorial +^^^^^^^^^^^^^^^^^^ + +What follows is a more technical tutorial for the mechanics and details of how +descriptors work. + + +Abstract +-------- + +Defines descriptors, summarizes the protocol, and shows how descriptors are +called. Provides an example showing how object relational mappings work. + +Learning about descriptors not only provides access to a larger toolset, it +creates a deeper understanding of how Python works. + + +Definition and introduction +--------------------------- + +In general, a descriptor is an attribute value that has one of the methods in +the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, +and :meth:`__delete__`. If any of those methods are defined for an +attribute, it is said to be a :term:`descriptor`. + +The default behavior for attribute access is to get, set, or delete the +attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain +starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and +continuing through the method resolution order of ``type(a)``. If the +looked-up value is an object defining one of the descriptor methods, then Python +may override the default behavior and invoke the descriptor method instead. +Where this occurs in the precedence chain depends on which descriptor methods +were defined. + +Descriptors are a powerful, general purpose protocol. They are the mechanism +behind properties, methods, static methods, class methods, and +:func:`super()`. They are used throughout Python itself. Descriptors +simplify the underlying C code and offer a flexible set of new tools for +everyday Python programs. + + +Descriptor protocol +------------------- + +``descr.__get__(self, obj, type=None) -> value`` + +``descr.__set__(self, obj, value) -> None`` + +``descr.__delete__(self, obj) -> None`` + +That is all there is to it. Define any of these methods and an object is +considered a descriptor and can override default behavior upon being looked up +as an attribute. + +If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered +a data descriptor. Descriptors that only define :meth:`__get__` are called +non-data descriptors (they are often used for methods but other uses are +possible). + +Data and non-data descriptors differ in how overrides are calculated with +respect to entries in an instance's dictionary. If an instance's dictionary +has an entry with the same name as a data descriptor, the data descriptor +takes precedence. If an instance's dictionary has an entry with the same +name as a non-data descriptor, the dictionary entry takes precedence. + +To make a read-only data descriptor, define both :meth:`__get__` and +:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when +called. Defining the :meth:`__set__` method with an exception raising +placeholder is enough to make it a data descriptor. + + +Overview of descriptor invocation +--------------------------------- + +A descriptor can be called directly with ``desc.__get__(obj)`` or +``desc.__get__(None, cls)``. + +But it is more common for a descriptor to be invoked automatically from +attribute access. + +The expression ``obj.x`` looks up the attribute ``x`` in the chain of +namespaces for ``obj``. If the search finds a descriptor outside of the +instance ``__dict__``, its :meth:`__get__` method is invoked according to the +precedence rules listed below. + +The details of invocation depend on whether ``obj`` is an object, class, or +instance of super. + + +Invocation from an instance +--------------------------- + +Instance lookup scans through a chain of namespaces giving data descriptors +the highest priority, followed by instance variables, then non-data +descriptors, then class variables, and lastly :meth:`__getattr__` if it is +provided. + +If a descriptor is found for ``a.x``, then it is invoked with: +``desc.__get__(a, type(a))``. + +The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is +a pure Python equivalent: + +.. testcode:: + + def object_getattribute(obj, name): + "Emulate PyObject_GenericGetAttr() in Objects/object.c" + null = object() + objtype = type(obj) + cls_var = getattr(objtype, name, null) + descr_get = getattr(type(cls_var), '__get__', null) + if descr_get is not null: + if (hasattr(type(cls_var), '__set__') + or hasattr(type(cls_var), '__delete__')): + return descr_get(cls_var, obj, objtype) # data descriptor + if hasattr(obj, '__dict__') and name in vars(obj): + return vars(obj)[name] # instance variable + if descr_get is not null: + return descr_get(cls_var, obj, objtype) # non-data descriptor + if cls_var is not null: + return cls_var # class variable + raise AttributeError(name) + + +.. testcode:: + :hide: + + # Test the fidelity of object_getattribute() by comparing it with the + # normal object.__getattribute__(). The former will be accessed by + # square brackets and the latter by the dot operator. + + class Object: + + def __getitem__(obj, name): + try: + return object_getattribute(obj, name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + + class DualOperator(Object): + + x = 10 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + @property + def p3(self): + return 3 * self.x + + def m5(self, y): + return 5 * y + + def m7(self, y): + return 7 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + class DualOperatorWithSlots: + + __getitem__ = Object.__getitem__ + + __slots__ = ['z'] + + x = 15 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + def m5(self, y): + return 5 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + +.. doctest:: + :hide: + + >>> a = DualOperator(11) + >>> vars(a).update(p3 = '_p3', m7 = '_m7') + >>> a.x == a['x'] == 10 + True + >>> a.z == a['z'] == 11 + True + >>> a.p2 == a['p2'] == 20 + True + >>> a.p3 == a['p3'] == 30 + True + >>> a.m5(100) == a.m5(100) == 500 + True + >>> a.m7 == a['m7'] == '_m7' + True + >>> a.g == a['g'] == ('getattr_hook', a, 'g') + True + + >>> b = DualOperatorWithSlots(22) + >>> b.x == b['x'] == 15 + True + >>> b.z == b['z'] == 22 + True + >>> b.p2 == b['p2'] == 30 + True + >>> b.m5(200) == b['m5'](200) == 1000 + True + >>> b.g == b['g'] == ('getattr_hook', b, 'g') + True + + +Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` +directly. Instead, both the dot operator and the :func:`getattr` function +perform attribute lookup by way of a helper function: + +.. testcode:: + + def getattr_hook(obj, name): + "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" + try: + return obj.__getattribute__(name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + +So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` +raises :exc:`AttributeError` (either directly or in one of the descriptor calls). + +Also, if a user calls :meth:`object.__getattribute__` directly, the +:meth:`__getattr__` hook is bypassed entirely. + + +Invocation from a class +----------------------- + +The logic for a dotted lookup such as ``A.x`` is in +:meth:`type.__getattribute__`. The steps are similar to those for +:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +by a search through the class's :term:`method resolution order`. + +If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. + +The full C implementation can be found in :c:func:`type_getattro()` and +:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. + + +Invocation from super +--------------------- + +The logic for super's dotted lookup is in the :meth:`__getattribute__` method for +object returned by :class:`super()`. + +A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` +for the base class ``B`` immediately following ``A`` and then returns +``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned +unchanged. + +The full C implementation can be found in :c:func:`super_getattro()` in +:source:`Objects/typeobject.c`. A pure Python equivalent can be found in +`Guido's Tutorial +`_. + + +Summary of invocation logic +--------------------------- + +The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +methods for :class:`object`, :class:`type`, and :func:`super`. + +The important points to remember are: + +* Descriptors are invoked by the :meth:`__getattribute__` method. + +* Classes inherit this machinery from :class:`object`, :class:`type`, or + :func:`super`. + +* Overriding :meth:`__getattribute__` prevents automatic descriptor calls + because all the descriptor logic is in that method. + +* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make + different calls to :meth:`__get__`. The first includes the instance and may + include the class. The second puts in ``None`` for the instance and always + includes the class. + +* Data descriptors always override instance dictionaries. + +* Non-data descriptors may be overridden by instance dictionaries. + + +Automatic name notification +--------------------------- + +Sometimes it is desirable for a descriptor to know what class variable name it +was assigned to. When a new class is created, the :class:`type` metaclass +scans the dictionary of the new class. If any of the entries are descriptors +and if they define :meth:`__set_name__`, that method is called with two +arguments. The *owner* is the class where the descriptor is used, and the +*name* is the class variable the descriptor was assigned to. + +The implementation details are in :c:func:`type_new()` and +:c:func:`set_names()` in :source:`Objects/typeobject.c`. + +Since the update logic is in :meth:`type.__new__`, notifications only take +place at the time of class creation. If descriptors are added to the class +afterwards, :meth:`__set_name__` will need to be called manually. + + +ORM example +----------- + +The following code is simplified skeleton showing how data descriptors could +be used to implement an `object relational mapping +`_. + +The essential idea is that the data is stored in an external database. The +Python instances only hold keys to the database's tables. Descriptors take +care of lookups or updates: + +.. testcode:: + + class Field: + + def __set_name__(self, owner, name): + self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' + self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' + + def __get__(self, obj, objtype=None): + return conn.execute(self.fetch, [obj.key]).fetchone()[0] + + def __set__(self, obj, value): + conn.execute(self.store, [value, obj.key]) + conn.commit() + +We can use the :class:`Field` class to define `models +`_ that describe the schema for +each table in a database: + +.. testcode:: + + class Movie: + table = 'Movies' # Table name + key = 'title' # Primary key + director = Field() + year = Field() + + def __init__(self, key): + self.key = key + + class Song: + table = 'Music' + key = 'title' + artist = Field() + year = Field() + genre = Field() + + def __init__(self, key): + self.key = key + +To use the models, first connect to the database:: + + >>> import sqlite3 + >>> conn = sqlite3.connect('entertainment.db') + +An interactive session shows how data is retrieved from the database and how +it can be updated: + +.. testsetup:: + + song_data = [ + ('Country Roads', 'John Denver', 1972), + ('Me and Bobby McGee', 'Janice Joplin', 1971), + ('Coal Miners Daughter', 'Loretta Lynn', 1970), + ] + + movie_data = [ + ('Star Wars', 'George Lucas', 1977), + ('Jaws', 'Steven Spielberg', 1975), + ('Aliens', 'James Cameron', 1986), + ] + + import sqlite3 + + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE Music (title text, artist text, year integer);') + conn.execute('CREATE INDEX MusicNdx ON Music (title);') + conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) + conn.execute('CREATE TABLE Movies (title text, director text, year integer);') + conn.execute('CREATE INDEX MovieNdx ON Music (title);') + conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) + conn.commit() + +.. doctest:: + + >>> Movie('Star Wars').director + 'George Lucas' + >>> jaws = Movie('Jaws') + >>> f'Released in {jaws.year} by {jaws.director}' + 'Released in 1975 by Steven Spielberg' + + >>> Song('Country Roads').artist + 'John Denver' + + >>> Movie('Star Wars').director = 'J.J. Abrams' + >>> Movie('Star Wars').director + 'J.J. Abrams' + + +Pure Python Equivalents +^^^^^^^^^^^^^^^^^^^^^^^ + +The descriptor protocol is simple and offers exciting possibilities. Several +use cases are so common that they have been prepackaged into built-in tools. +Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are +all based on the descriptor protocol. + + +Properties +---------- + +Calling :func:`property` is a succinct way of building a data descriptor that +triggers a function call upon access to an attribute. Its signature is:: + + property(fget=None, fset=None, fdel=None, doc=None) -> property + +The documentation shows a typical use to define a managed attribute ``x``: + +.. testcode:: + + class C: + def getx(self): return self.__x + def setx(self, value): self.__x = value + def delx(self): del self.__x + x = property(getx, setx, delx, "I'm the 'x' property.") + +To see how :func:`property` is implemented in terms of the descriptor protocol, +here is a pure Python equivalent: + +.. testcode:: + + class Property: + "Emulate PyProperty_Type() in Objects/descrobject.c" + + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + if doc is None and fget is not None: + doc = fget.__doc__ + self.__doc__ = doc + self._name = '' + + def __set_name__(self, owner, name): + self._name = name + + def __get__(self, obj, objtype=None): + if obj is None: + return self + if self.fget is None: + raise AttributeError(f'unreadable attribute {self._name}') + return self.fget(obj) + + def __set__(self, obj, value): + if self.fset is None: + raise AttributeError(f"can't set attribute {self._name}") + self.fset(obj, value) + + def __delete__(self, obj): + if self.fdel is None: + raise AttributeError(f"can't delete attribute {self._name}") + self.fdel(obj) + + def getter(self, fget): + prop = type(self)(fget, self.fset, self.fdel, self.__doc__) + prop._name = self._name + return prop + + def setter(self, fset): + prop = type(self)(self.fget, fset, self.fdel, self.__doc__) + prop._name = self._name + return prop + + def deleter(self, fdel): + prop = type(self)(self.fget, self.fset, fdel, self.__doc__) + prop._name = self._name + return prop + +.. testcode:: + :hide: + + # Verify the Property() emulation + + class CC: + def getx(self): + return self.__x + def setx(self, value): + self.__x = value + def delx(self): + del self.__x + x = Property(getx, setx, delx, "I'm the 'x' property.") + + # Now do it again but use the decorator style + + class CCC: + @Property + def x(self): + return self.__x + @x.setter + def x(self, value): + self.__x = value + @x.deleter + def x(self): + del self.__x + + +.. doctest:: + :hide: + + >>> cc = CC() + >>> hasattr(cc, 'x') + False + >>> cc.x = 33 + >>> cc.x + 33 + >>> del cc.x + >>> hasattr(cc, 'x') + False + + >>> ccc = CCC() + >>> hasattr(ccc, 'x') + False + >>> ccc.x = 333 + >>> ccc.x == 333 + True + >>> del ccc.x + >>> hasattr(ccc, 'x') + False + +The :func:`property` builtin helps whenever a user interface has granted +attribute access and then subsequent changes require the intervention of a +method. + +For instance, a spreadsheet class may grant access to a cell value through +``Cell('b10').value``. Subsequent improvements to the program require the cell +to be recalculated on every access; however, the programmer does not want to +affect existing client code accessing the attribute directly. The solution is +to wrap access to the value attribute in a property data descriptor: + +.. testcode:: + + class Cell: + ... + + @property + def value(self): + "Recalculate the cell before returning value" + self.recalc() + return self._value + +Either the built-in :func:`property` or our :func:`Property` equivalent would +work in this example. + + +Functions and methods +--------------------- + +Python's object oriented features are built upon a function based environment. +Using non-data descriptors, the two are merged seamlessly. + +Functions stored in class dictionaries get turned into methods when invoked. +Methods only differ from regular functions in that the object instance is +prepended to the other arguments. By convention, the instance is called +*self* but could be called *this* or any other variable name. + +Methods can be created manually with :class:`types.MethodType` which is +roughly equivalent to: + +.. testcode:: + + class MethodType: + "Emulate Py_MethodType in Objects/classobject.c" + + def __init__(self, func, obj): + self.__func__ = func + self.__self__ = obj + + def __call__(self, *args, **kwargs): + func = self.__func__ + obj = self.__self__ + return func(obj, *args, **kwargs) + +To support automatic creation of methods, functions include the +:meth:`__get__` method for binding methods during attribute access. This +means that functions are non-data descriptors that return bound methods +during dotted lookup from an instance. Here's how it works: + +.. testcode:: + + class Function: + ... + + def __get__(self, obj, objtype=None): + "Simulate func_descr_get() in Objects/funcobject.c" + if obj is None: + return self + return MethodType(self, obj) + +Running the following class in the interpreter shows how the function +descriptor works in practice: + +.. testcode:: + + class D: + def f(self, x): + return x + +The function has a :term:`qualified name` attribute to support introspection: + +.. doctest:: + + >>> D.f.__qualname__ + 'D.f' + +Accessing the function through the class dictionary does not invoke +:meth:`__get__`. Instead, it just returns the underlying function object:: + + >>> D.__dict__['f'] + + +Dotted access from a class calls :meth:`__get__` which just returns the +underlying function unchanged:: + + >>> D.f + + +The interesting behavior occurs during dotted access from an instance. The +dotted lookup calls :meth:`__get__` which returns a bound method object:: + + >>> d = D() + >>> d.f + > + +Internally, the bound method stores the underlying function and the bound +instance:: + + >>> d.f.__func__ + + + >>> d.f.__self__ + <__main__.D object at 0x1012e1f98> + +If you have ever wondered where *self* comes from in regular methods or where +*cls* comes from in class methods, this is it! + + +Static methods +-------------- + +Non-data descriptors provide a simple mechanism for variations on the usual +patterns of binding functions into methods. + +To recap, functions have a :meth:`__get__` method so that they can be converted +to a method when accessed as attributes. The non-data descriptor transforms an +``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` +becomes ``f(*args)``. + +This chart summarizes the binding and its two most useful variants: + + +-----------------+----------------------+------------------+ + | Transformation | Called from an | Called from a | + | | object | class | + +=================+======================+==================+ + | function | f(obj, \*args) | f(\*args) | + +-----------------+----------------------+------------------+ + | staticmethod | f(\*args) | f(\*args) | + +-----------------+----------------------+------------------+ + | classmethod | f(type(obj), \*args) | f(cls, \*args) | + +-----------------+----------------------+------------------+ + +Static methods return the underlying function without changes. Calling either +``c.f`` or ``C.f`` is the equivalent of a direct lookup into +``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a +result, the function becomes identically accessible from either an object or a +class. + +Good candidates for static methods are methods that do not reference the +``self`` variable. + +For instance, a statistics package may include a container class for +experimental data. The class provides normal methods for computing the average, +mean, median, and other descriptive statistics that depend on the data. However, +there may be useful functions which are conceptually related but do not depend +on the data. For instance, ``erf(x)`` is handy conversion routine that comes up +in statistical work but does not directly depend on a particular dataset. +It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or +``Sample.erf(1.5) --> .9332``. + +Since static methods return the underlying function with no changes, the +example calls are unexciting: + +.. testcode:: + + class E: + @staticmethod + def f(x): + print(x) + +.. doctest:: + + >>> E.f(3) + 3 + >>> E().f(3) + 3 + +Using the non-data descriptor protocol, a pure Python version of +:func:`staticmethod` would look like this: + +.. doctest:: + + class StaticMethod: + "Emulate PyStaticMethod_Type() in Objects/funcobject.c" + + def __init__(self, f): + self.f = f + + def __get__(self, obj, objtype=None): + return self.f + + +Class methods +------------- + +Unlike static methods, class methods prepend the class reference to the +argument list before calling the function. This format is the same +for whether the caller is an object or a class: + +.. testcode:: + + class F: + @classmethod + def f(cls, x): + return cls.__name__, x + +.. doctest:: + + >>> F.f(3) + ('F', 3) + >>> F().f(3) + ('F', 3) + +This behavior is useful whenever the method only needs to have a class +reference and does not rely on data stored in a specific instance. One use for +class methods is to create alternate class constructors. For example, the +classmethod :func:`dict.fromkeys` creates a new dictionary from a list of +keys. The pure Python equivalent is: + +.. testcode:: + + class Dict(dict): + @classmethod + def fromkeys(cls, iterable, value=None): + "Emulate dict_fromkeys() in Objects/dictobject.c" + d = cls() + for key in iterable: + d[key] = value + return d + +Now a new dictionary of unique keys can be constructed like this: + +.. doctest:: + + >>> d = Dict.fromkeys('abracadabra') + >>> type(d) is Dict + True + >>> d + {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} + +Using the non-data descriptor protocol, a pure Python version of +:func:`classmethod` would look like this: + +.. testcode:: + + class ClassMethod: + "Emulate PyClassMethod_Type() in Objects/funcobject.c" + + def __init__(self, f): + self.f = f + + def __get__(self, obj, cls=None): + if cls is None: + cls = type(obj) + if hasattr(obj, '__get__'): + return self.f.__get__(cls) + return MethodType(self.f, cls) + +.. testcode:: + :hide: + + # Verify the emulation works + class T: + @ClassMethod + def cm(cls, x, y): + return (cls, x, y) + +.. doctest:: + :hide: + + >>> T.cm(11, 22) + (, 11, 22) + + # Also call it from an instance + >>> t = T() + >>> t.cm(11, 22) + (, 11, 22) + +The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and +makes it possible for :func:`classmethod` to support chained decorators. +For example, a classmethod and property could be chained together: + +.. testcode:: + + class G: + @classmethod + @property + def __doc__(cls): + return f'A doc for {cls.__name__!r}' + +.. doctest:: + + >>> G.__doc__ + "A doc for 'G'" + + +Member objects and __slots__ +---------------------------- + +When a class defines ``__slots__``, it replaces instance dictionaries with a +fixed-length array of slot values. From a user point of view that has +several effects: + +1. Provides immediate detection of bugs due to misspelled attribute +assignments. Only attribute names specified in ``__slots__`` are allowed: + +.. testcode:: + + class Vehicle: + __slots__ = ('id_number', 'make', 'model') + +.. doctest:: + + >>> auto = Vehicle() + >>> auto.id_nubmer = 'VYE483814LQEX' + Traceback (most recent call last): + ... + AttributeError: 'Vehicle' object has no attribute 'id_nubmer' + +2. Helps create immutable objects where descriptors manage access to private +attributes stored in ``__slots__``: + +.. testcode:: + + class Immutable: + + __slots__ = ('_dept', '_name') # Replace the instance dictionary + + def __init__(self, dept, name): + self._dept = dept # Store to private attribute + self._name = name # Store to private attribute + + @property # Read-only descriptor + def dept(self): + return self._dept + + @property + def name(self): # Read-only descriptor + return self._name + +.. doctest:: + + >>> mark = Immutable('Botany', 'Mark Watney') + >>> mark.dept + 'Botany' + >>> mark.dept = 'Space Pirate' + Traceback (most recent call last): + ... + AttributeError: can't set attribute + >>> mark.location = 'Mars' + Traceback (most recent call last): + ... + AttributeError: 'Immutable' object has no attribute 'location' + +3. Saves memory. On a 64-bit Linux build, an instance with two attributes +takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight +design pattern `_ likely only +matters when a large number of instances are going to be created. + +4. Improves speed. Reading instance variables is 35% faster with +``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). + +5. Blocks tools like :func:`functools.cached_property` which require an +instance dictionary to function correctly: + +.. testcode:: + + from functools import cached_property + + class CP: + __slots__ = () # Eliminates the instance dict + + @cached_property # Requires an instance dict + def pi(self): + return 4 * sum((-1.0)**n / (2.0*n + 1.0) + for n in reversed(range(100_000))) + +.. doctest:: + + >>> CP().pi + Traceback (most recent call last): + ... + TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. + +It is not possible to create an exact drop-in pure Python version of +``__slots__`` because it requires direct access to C structures and control +over object memory allocation. However, we can build a mostly faithful +simulation where the actual C structure for slots is emulated by a private +``_slotvalues`` list. Reads and writes to that private structure are managed +by member descriptors: + +.. testcode:: + + null = object() + + class Member: + + def __init__(self, name, clsname, offset): + 'Emulate PyMemberDef in Include/structmember.h' + # Also see descr_new() in Objects/descrobject.c + self.name = name + self.clsname = clsname + self.offset = offset + + def __get__(self, obj, objtype=None): + 'Emulate member_get() in Objects/descrobject.c' + # Also see PyMember_GetOne() in Python/structmember.c + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + return value + + def __set__(self, obj, value): + 'Emulate member_set() in Objects/descrobject.c' + obj._slotvalues[self.offset] = value + + def __delete__(self, obj): + 'Emulate member_delete() in Objects/descrobject.c' + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + obj._slotvalues[self.offset] = null + + def __repr__(self): + 'Emulate member_repr() in Objects/descrobject.c' + return f'' + +The :meth:`type.__new__` method takes care of adding member objects to class +variables: + +.. testcode:: + + class Type(type): + 'Simulate how the type metaclass adds member objects for slots' + + def __new__(mcls, clsname, bases, mapping): + 'Emuluate type_new() in Objects/typeobject.c' + # type_new() calls PyTypeReady() which calls add_methods() + slot_names = mapping.get('slot_names', []) + for offset, name in enumerate(slot_names): + mapping[name] = Member(name, clsname, offset) + return type.__new__(mcls, clsname, bases, mapping) + +The :meth:`object.__new__` method takes care of creating instances that have +slots instead of an instance dictionary. Here is a rough simulation in pure +Python: + +.. testcode:: + + class Object: + 'Simulate how object.__new__() allocates memory for __slots__' + + def __new__(cls, *args): + 'Emulate object_new() in Objects/typeobject.c' + inst = super().__new__(cls) + if hasattr(cls, 'slot_names'): + empty_slots = [null] * len(cls.slot_names) + object.__setattr__(inst, '_slotvalues', empty_slots) + return inst + + def __setattr__(self, name, value): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__setattr__(name, value) + + def __delattr__(self, name): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__delattr__(name) + +To use the simulation in a real class, just inherit from :class:`Object` and +set the :term:`metaclass` to :class:`Type`: + +.. testcode:: + + class H(Object, metaclass=Type): + 'Instance variables stored in slots' + + slot_names = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + +At this point, the metaclass has loaded member objects for *x* and *y*:: + + >>> from pprint import pp + >>> pp(dict(vars(H))) + {'__module__': '__main__', + '__doc__': 'Instance variables stored in slots', + 'slot_names': ['x', 'y'], + '__init__': , + 'x': , + 'y': } + +.. doctest:: + :hide: + + # We test this separately because the preceding section is not + # doctestable due to the hex memory address for the __init__ function + >>> isinstance(vars(H)['x'], Member) + True + >>> isinstance(vars(H)['y'], Member) + True + +When instances are created, they have a ``slot_values`` list where the +attributes are stored: + +.. doctest:: + + >>> h = H(10, 20) + >>> vars(h) + {'_slotvalues': [10, 20]} + >>> h.x = 55 + >>> vars(h) + {'_slotvalues': [55, 20]} + +Misspelled or unassigned attributes will raise an exception: + +.. doctest:: + + >>> h.xz + Traceback (most recent call last): + ... + AttributeError: 'H' object has no attribute 'xz' + +.. doctest:: + :hide: + + # Examples for deleted attributes are not shown because this section + # is already a bit lengthy. We still test that code here. + >>> del h.x + >>> hasattr(h, 'x') + False + + # Also test the code for uninitialized slots + >>> class HU(Object, metaclass=Type): + ... slot_names = ['x', 'y'] + ... + >>> hu = HU() + >>> hasattr(hu, 'x') + False + >>> hasattr(hu, 'y') + False diff --git a/Doc/library/_thread.rst.bak b/Doc/library/_thread.rst.bak new file mode 100644 index 000000000000000..bd653ab32bb9c43 --- /dev/null +++ b/Doc/library/_thread.rst.bak @@ -0,0 +1,215 @@ +:mod:`_thread` --- Low-level threading API +========================================== + +.. module:: _thread + :synopsis: Low-level threading API. + +.. index:: + single: light-weight processes + single: processes, light-weight + single: binary semaphores + single: semaphores, binary + +-------------- + +This module provides low-level primitives for working with multiple threads +(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of +control sharing their global data space. For synchronization, simple locks +(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. +The :mod:`threading` module provides an easier to use and higher-level +threading API built on top of this module. + +.. index:: + single: pthreads + pair: threads; POSIX + +.. versionchanged:: 3.7 + This module used to be optional, it is now always available. + +This module defines the following constants and functions: + +.. exception:: error + + Raised on thread-specific errors. + + .. versionchanged:: 3.3 + This is now a synonym of the built-in :exc:`RuntimeError`. + + +.. data:: LockType + + This is the type of lock objects. + + +.. function:: start_new_thread(function, args[, kwargs]) + + Start a new thread and return its identifier. The thread executes the + function *function* with the argument list *args* (which must be a tuple). + The optional *kwargs* argument specifies a dictionary of keyword arguments. + + When the function returns, the thread silently exits. + + When the function terminates with an unhandled exception, + :func:`sys.unraisablehook` is called to handle the exception. The *object* + attribute of the hook argument is *function*. By default, a stack trace is + printed and then the thread exits (but other threads continue to run). + + When the function raises a :exc:`SystemExit` exception, it is silently + ignored. + + .. versionchanged:: 3.8 + :func:`sys.unraisablehook` is now used to handle unhandled exceptions. + + +.. function:: interrupt_main() + + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. + + +.. function:: exit() + + Raise the :exc:`SystemExit` exception. When not caught, this will cause the + thread to exit silently. + +.. + function:: exit_prog(status) + + Exit all threads and report the value of the integer argument + *status* as the exit status of the entire program. + **Caveat:** code in pending :keyword:`finally` clauses, in this thread + or in other threads, is not executed. + + +.. function:: allocate_lock() + + Return a new lock object. Methods of locks are described below. The lock is + initially unlocked. + + +.. function:: get_ident() + + Return the 'thread identifier' of the current thread. This is a nonzero + integer. Its value has no direct meaning; it is intended as a magic cookie to + be used e.g. to index a dictionary of thread-specific data. Thread identifiers + may be recycled when a thread exits and another thread is created. + + +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + +.. function:: stack_size([size]) + + Return the thread stack size used when creating new threads. The optional + *size* argument specifies the stack size to be used for subsequently created + threads, and must be 0 (use platform or configured default) or a positive + integer value of at least 32,768 (32 KiB). If *size* is not specified, + 0 is used. If changing the thread stack size is + unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is + invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB + is currently the minimum supported stack size value to guarantee sufficient + stack space for the interpreter itself. Note that some platforms may have + particular restrictions on values for the stack size, such as requiring a + minimum stack size > 32 KiB or requiring allocation in multiples of the system + memory page size - platform documentation should be referred to for more + information (4 KiB pages are common; using multiples of 4096 for the stack size is + the suggested approach in the absence of more specific information). + + .. availability:: Windows, systems with POSIX threads. + + +.. data:: TIMEOUT_MAX + + The maximum value allowed for the *timeout* parameter of + :meth:`Lock.acquire`. Specifying a timeout greater than this value will + raise an :exc:`OverflowError`. + + .. versionadded:: 3.2 + + +Lock objects have the following methods: + + +.. method:: lock.acquire(waitflag=1, timeout=-1) + + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). + + If the integer *waitflag* argument is present, the action depends on its + value: if it is zero, the lock is only acquired if it can be acquired + immediately without waiting, while if it is nonzero, the lock is acquired + unconditionally as above. + + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *waitflag* is zero. + + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. + + .. versionchanged:: 3.2 + The *timeout* parameter is new. + + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. + + +.. method:: lock.release() + + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. + + +.. method:: lock.locked() + + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. + +In addition to these methods, lock objects can also be used via the +:keyword:`with` statement, e.g.:: + + import _thread + + a_lock = _thread.allocate_lock() + + with a_lock: + print("a_lock is locked while this executes") + +**Caveats:** + + .. index:: module: signal + +* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` + exception will be received by an arbitrary thread. (When the :mod:`signal` + module is available, interrupts always go to the main thread.) + +* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is + equivalent to calling :func:`_thread.exit`. + +* It is not possible to interrupt the :meth:`acquire` method on a lock --- the + :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. + +* When the main thread exits, it is system defined whether the other threads + survive. On most systems, they are killed without executing + :keyword:`try` ... :keyword:`finally` clauses or executing object + destructors. + +* When the main thread exits, it does not do any of its usual cleanup (except + that :keyword:`try` ... :keyword:`finally` clauses are honored), and the + standard I/O files are not flushed. + diff --git a/Doc/library/ast.rst.bak b/Doc/library/ast.rst.bak new file mode 100644 index 000000000000000..b5e1be826911575 --- /dev/null +++ b/Doc/library/ast.rst.bak @@ -0,0 +1,1993 @@ +:mod:`ast` --- Abstract Syntax Trees +==================================== + +.. module:: ast + :synopsis: Abstract Syntax Tree classes and manipulation. + +.. sectionauthor:: Martin v. Löwis +.. sectionauthor:: Georg Brandl + +.. testsetup:: + + import ast + +**Source code:** :source:`Lib/ast.py` + +-------------- + +The :mod:`ast` module helps Python applications to process trees of the Python +abstract syntax grammar. The abstract syntax itself might change with each +Python release; this module helps to find out programmatically what the current +grammar looks like. + +An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as +a flag to the :func:`compile` built-in function, or using the :func:`parse` +helper provided in this module. The result will be a tree of objects whose +classes all inherit from :class:`ast.AST`. An abstract syntax tree can be +compiled into a Python code object using the built-in :func:`compile` function. + + +.. _abstract-grammar: + +Abstract Grammar +---------------- + +The abstract grammar is currently defined as follows: + +.. literalinclude:: ../../Parser/Python.asdl + :language: asdl + + +Node classes +------------ + +.. class:: AST + + This is the base of all AST node classes. The actual node classes are + derived from the :file:`Parser/Python.asdl` file, which is reproduced + :ref:`below `. They are defined in the :mod:`_ast` C + module and re-exported in :mod:`ast`. + + There is one class defined for each left-hand side symbol in the abstract + grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, + there is one class defined for each constructor on the right-hand side; these + classes inherit from the classes for the left-hand side trees. For example, + :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules + with alternatives (aka "sums"), the left-hand side class is abstract: only + instances of specific constructor nodes are ever created. + + .. index:: single: ? (question mark); in AST grammar + .. index:: single: * (asterisk); in AST grammar + + .. attribute:: _fields + + Each concrete class has an attribute :attr:`_fields` which gives the names + of all child nodes. + + Each instance of a concrete class has one attribute for each child node, + of the type as defined in the grammar. For example, :class:`ast.BinOp` + instances have an attribute :attr:`left` of type :class:`ast.expr`. + + If these attributes are marked as optional in the grammar (using a + question mark), the value might be ``None``. If the attributes can have + zero-or-more values (marked with an asterisk), the values are represented + as Python lists. All possible attributes must be present and have valid + values when compiling an AST with :func:`compile`. + + .. attribute:: lineno + col_offset + end_lineno + end_col_offset + + Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have + :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and + :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` + are the first and last line numbers of source text span (1-indexed so the + first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` + are the corresponding UTF-8 byte offsets of the first and last tokens that + generated the node. The UTF-8 offset is recorded because the parser uses + UTF-8 internally. + + Note that the end positions are not required by the compiler and are + therefore optional. The end offset is *after* the last symbol, for example + one can get the source segment of a one-line expression node using + ``source_line[node.col_offset : node.end_col_offset]``. + + The constructor of a class :class:`ast.T` parses its arguments as follows: + + * If there are positional arguments, there must be as many as there are items + in :attr:`T._fields`; they will be assigned as attributes of these names. + * If there are keyword arguments, they will set the attributes of the same + names to the given values. + + For example, to create and populate an :class:`ast.UnaryOp` node, you could + use :: + + node = ast.UnaryOp() + node.op = ast.USub() + node.operand = ast.Constant() + node.operand.value = 5 + node.operand.lineno = 0 + node.operand.col_offset = 0 + node.lineno = 0 + node.col_offset = 0 + + or the more compact :: + + node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), + lineno=0, col_offset=0) + +.. versionchanged:: 3.8 + + Class :class:`ast.Constant` is now used for all constants. + +.. versionchanged:: 3.9 + + Simple indices are represented by their value, extended slices are + represented as tuples. + +.. deprecated:: 3.8 + + Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, + :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, + but they will be removed in future Python releases. In the meantime, + instantiating them will return an instance of a different class. + +.. deprecated:: 3.9 + + Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + available, but they will be removed in future Python releases. + In the meantime, instantiating them will return an instance of + a different class. + +.. note:: + The descriptions of the specific node classes displayed here + were initially adapted from the fantastic `Green Tree + Snakes `__ project and + all its contributors. + +Literals +^^^^^^^^ + +.. class:: Constant(value) + + A constant value. The ``value`` attribute of the ``Constant`` literal contains the + Python object it represents. The values represented can be simple types + such as a number, string or ``None``, but also immutable container types + (tuples and frozensets) if all of their elements are constant. + + .. doctest:: + + >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) + Expression( + body=Constant(value=123)) + + +.. class:: FormattedValue(value, conversion, format_spec) + + Node representing a single formatting field in an f-string. If the string + contains a single formatting field and nothing else the node can be + isolated otherwise it appears in :class:`JoinedStr`. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``conversion`` is an integer: + + * -1: no formatting + * 115: ``!s`` string formatting + * 114: ``!r`` repr formatting + * 97: ``!a`` ascii formatting + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + +.. class:: JoinedStr(values) + + An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) + Expression( + body=JoinedStr( + values=[ + Constant(value='sin('), + FormattedValue( + value=Name(id='a', ctx=Load()), + conversion=-1), + Constant(value=') is '), + FormattedValue( + value=Call( + func=Name(id='sin', ctx=Load()), + args=[ + Name(id='a', ctx=Load())], + keywords=[]), + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='.3')]))])) + + +.. class:: List(elts, ctx) + Tuple(elts, ctx) + + A list or tuple. ``elts`` holds a list of nodes representing the elements. + ``ctx`` is :class:`Store` if the container is an assignment target (i.e. + ``(x,y)=something``), and :class:`Load` otherwise. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) + Expression( + body=List( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) + Expression( + body=Tuple( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + + +.. class:: Set(elts) + + A set. ``elts`` holds a list of nodes representing the set's elements. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) + Expression( + body=Set( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + + +.. class:: Dict(keys, values) + + A dictionary. ``keys`` and ``values`` hold lists of nodes representing the + keys and the values respectively, in matching order (what would be returned + when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). + + When doing dictionary unpacking using dictionary literals the expression to be + expanded goes in the ``values`` list, with a ``None`` at the corresponding + position in ``keys``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) + Expression( + body=Dict( + keys=[ + Constant(value='a'), + None], + values=[ + Constant(value=1), + Name(id='d', ctx=Load())])) + + +Variables +^^^^^^^^^ + +.. class:: Name(id, ctx) + + A variable name. ``id`` holds the name as a string, and ``ctx`` is one of + the following types. + + +.. class:: Load() + Store() + Del() + + Variable references can be used to load the value of a variable, to assign + a new value to it, or to delete it. Variable references are given a context + to distinguish these cases. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a'), indent=4)) + Module( + body=[ + Expr( + value=Name(id='a', ctx=Load()))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a = 1'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('del a'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='a', ctx=Del())])], + type_ignores=[]) + + +.. class:: Starred(value, ctx) + + A ``*var`` variable reference. ``value`` holds the variable, typically a + :class:`Name` node. This type must be used when building a :class:`Call` + node with ``*args``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Starred( + value=Name(id='b', ctx=Store()), + ctx=Store())], + ctx=Store())], + value=Name(id='it', ctx=Load()))], + type_ignores=[]) + + +Expressions +^^^^^^^^^^^ + +.. class:: Expr(value) + + When an expression, such as a function call, appears as a statement by itself + with its return value not used or stored, it is wrapped in this container. + ``value`` holds one of the other nodes in this section, a :class:`Constant`, a + :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('-a'), indent=4)) + Module( + body=[ + Expr( + value=UnaryOp( + op=USub(), + operand=Name(id='a', ctx=Load())))], + type_ignores=[]) + + +.. class:: UnaryOp(op, operand) + + A unary operation. ``op`` is the operator, and ``operand`` any expression + node. + + +.. class:: UAdd + USub + Not + Invert + + Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` + is the ``~`` operator. + + .. doctest:: + + >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) + Expression( + body=UnaryOp( + op=Not(), + operand=Name(id='x', ctx=Load()))) + + +.. class:: BinOp(left, op, right) + + A binary operation (like addition or division). ``op`` is the operator, and + ``left`` and ``right`` are any expression nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) + Expression( + body=BinOp( + left=Name(id='x', ctx=Load()), + op=Add(), + right=Name(id='y', ctx=Load()))) + + +.. class:: Add + Sub + Mult + Div + FloorDiv + Mod + Pow + LShift + RShift + BitOr + BitXor + BitAnd + MatMult + + Binary operator tokens. + + +.. class:: BoolOp(op, values) + + A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. + ``values`` are the values involved. Consecutive operations with the same + operator, such as ``a or b or c``, are collapsed into one node with several + values. + + This doesn't include ``not``, which is a :class:`UnaryOp`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) + Expression( + body=BoolOp( + op=Or(), + values=[ + Name(id='x', ctx=Load()), + Name(id='y', ctx=Load())])) + + +.. class:: And + Or + + Boolean operator tokens. + + +.. class:: Compare(left, ops, comparators) + + A comparison of two or more values. ``left`` is the first value in the + comparison, ``ops`` the list of operators, and ``comparators`` the list + of values after the first element in the comparison. + + .. doctest:: + + >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) + Expression( + body=Compare( + left=Constant(value=1), + ops=[ + LtE(), + Lt()], + comparators=[ + Name(id='a', ctx=Load()), + Constant(value=10)])) + + +.. class:: Eq + NotEq + Lt + LtE + Gt + GtE + Is + IsNot + In + NotIn + + Comparison operator tokens. + + +.. class:: Call(func, args, keywords, starargs, kwargs) + + A function call. ``func`` is the function, which will often be a + :class:`Name` or :class:`Attribute` object. Of the arguments: + + * ``args`` holds a list of the arguments passed by position. + * ``keywords`` holds a list of :class:`keyword` objects representing + arguments passed by keyword. + + When creating a ``Call`` node, ``args`` and ``keywords`` are required, but + they can be empty lists. ``starargs`` and ``kwargs`` are optional. + + .. doctest:: + + >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) + Expression( + body=Call( + func=Name(id='func', ctx=Load()), + args=[ + Name(id='a', ctx=Load()), + Starred( + value=Name(id='d', ctx=Load()), + ctx=Load())], + keywords=[ + keyword( + arg='b', + value=Name(id='c', ctx=Load())), + keyword( + value=Name(id='e', ctx=Load()))])) + + +.. class:: keyword(arg, value) + + A keyword argument to a function call or class definition. ``arg`` is a raw + string of the parameter name, ``value`` is a node to pass in. + + +.. class:: IfExp(test, body, orelse) + + An expression such as ``a if b else c``. Each field holds a single node, so + in the following example, all three are :class:`Name` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) + Expression( + body=IfExp( + test=Name(id='b', ctx=Load()), + body=Name(id='a', ctx=Load()), + orelse=Name(id='c', ctx=Load()))) + + +.. class:: Attribute(value, attr, ctx) + + Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a + :class:`Name`. ``attr`` is a bare string giving the name of the attribute, + and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how + the attribute is acted on. + + .. doctest:: + + >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) + Expression( + body=Attribute( + value=Name(id='snake', ctx=Load()), + attr='colour', + ctx=Load())) + + +.. class:: NamedExpr(target, value) + + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) + Expression( + body=NamedExpr( + target=Name(id='x', ctx=Store()), + value=Constant(value=4))) + + +Subscripting +~~~~~~~~~~~~ + +.. class:: Subscript(value, slice, ctx) + + A subscript, such as ``l[1]``. ``value`` is the subscripted object + (usually sequence or mapping). ``slice`` is an index, slice or key. + It can be a :class:`Tuple` and contain a :class:`Slice`. + ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` + according to the action performed with the subscript. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Tuple( + elts=[ + Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + Constant(value=3)], + ctx=Load()), + ctx=Load())) + + +.. class:: Slice(lower, upper, step) + + Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). + Can occur only inside the *slice* field of :class:`Subscript`, either + directly or as an element of :class:`Tuple`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + ctx=Load())) + + +Comprehensions +~~~~~~~~~~~~~~ + +.. class:: ListComp(elt, generators) + SetComp(elt, generators) + GeneratorExp(elt, generators) + DictComp(key, value, generators) + + List and set comprehensions, generator expressions, and dictionary + comprehensions. ``elt`` (or ``key`` and ``value``) is a single node + representing the part that will be evaluated for each item. + + ``generators`` is a list of :class:`comprehension` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + Expression( + body=ListComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + Expression( + body=DictComp( + key=Name(id='x', ctx=Load()), + value=BinOp( + left=Name(id='x', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + Expression( + body=SetComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + + +.. class:: comprehension(target, iter, ifs, is_async) + + One ``for`` clause in a comprehension. ``target`` is the reference to use for + each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` + is the object to iterate over. ``ifs`` is a list of test expressions: each + ``for`` clause can have multiple ``ifs``. + + ``is_async`` indicates a comprehension is asynchronous (using an + ``async for`` instead of ``for``). The value is an integer (0 or 1). + + .. doctest:: + + >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), + ... indent=4)) # Multiple comprehensions in one. + Expression( + body=ListComp( + elt=Call( + func=Name(id='ord', ctx=Load()), + args=[ + Name(id='c', ctx=Load())], + keywords=[]), + generators=[ + comprehension( + target=Name(id='line', ctx=Store()), + iter=Name(id='file', ctx=Load()), + ifs=[], + is_async=0), + comprehension( + target=Name(id='c', ctx=Store()), + iter=Name(id='line', ctx=Load()), + ifs=[], + is_async=0)])) + + >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), + ... indent=4)) # generator comprehension + Expression( + body=GeneratorExp( + elt=BinOp( + left=Name(id='n', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='n', ctx=Store()), + iter=Name(id='it', ctx=Load()), + ifs=[ + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Lt()], + comparators=[ + Constant(value=10)])], + is_async=0)])) + + >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), + ... indent=4)) # Async comprehension + Expression( + body=ListComp( + elt=Name(id='i', ctx=Load()), + generators=[ + comprehension( + target=Name(id='i', ctx=Store()), + iter=Name(id='soc', ctx=Load()), + ifs=[], + is_async=1)])) + +Statements +^^^^^^^^^^ + +.. class:: Assign(targets, value, type_comment) + + An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. + + Multiple nodes in ``targets`` represents assigning the same value to each. + Unpacking is represented by putting a :class:`Tuple` or :class:`List` + within ``targets``. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + ctx=Store())], + value=Name(id='c', ctx=Load()))], + type_ignores=[]) + + +.. class:: AnnAssign(target, annotation, value, simple) + + An assignment with a type annotation. ``target`` is a single node and can + be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` + node. ``value`` is a single optional node. ``simple`` is a boolean integer + set to True for a :class:`Name` node in ``target`` that do not appear in + between parenthesis and are hence pure names and not expressions. + + .. doctest:: + + >>> print(ast.dump(ast.parse('c: int'), indent=4)) + Module( + body=[ + AnnAssign( + target=Name(id='c', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=1)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis + Module( + body=[ + AnnAssign( + target=Name(id='a', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + value=Constant(value=1), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation + Module( + body=[ + AnnAssign( + target=Attribute( + value=Name(id='a', ctx=Load()), + attr='b', + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation + Module( + body=[ + AnnAssign( + target=Subscript( + value=Name(id='a', ctx=Load()), + slice=Constant(value=1), + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + +.. class:: AugAssign(target, op, value) + + Augmented assignment, such as ``a += 1``. In the following example, + ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` + context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with + value for 1. + + The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, + unlike the targets of :class:`Assign`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x += 2'), indent=4)) + Module( + body=[ + AugAssign( + target=Name(id='x', ctx=Store()), + op=Add(), + value=Constant(value=2))], + type_ignores=[]) + + +.. class:: Raise(exc, cause) + + A ``raise`` statement. ``exc`` is the exception object to be raised, normally a + :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. + ``cause`` is the optional part for ``y`` in ``raise x from y``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) + Module( + body=[ + Raise( + exc=Name(id='x', ctx=Load()), + cause=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Assert(test, msg) + + An assertion. ``test`` holds the condition, such as a :class:`Compare` node. + ``msg`` holds the failure message. + + .. doctest:: + + >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) + Module( + body=[ + Assert( + test=Name(id='x', ctx=Load()), + msg=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Delete(targets) + + Represents a ``del`` statement. ``targets`` is a list of nodes, such as + :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='x', ctx=Del()), + Name(id='y', ctx=Del()), + Name(id='z', ctx=Del())])], + type_ignores=[]) + + +.. class:: Pass() + + A ``pass`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('pass'), indent=4)) + Module( + body=[ + Pass()], + type_ignores=[]) + + +Other statements which are only applicable inside functions or loops are +described in other sections. + +Imports +~~~~~~~ + +.. class:: Import(names) + + An import statement. ``names`` is a list of :class:`alias` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) + Module( + body=[ + Import( + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')])], + type_ignores=[]) + + +.. class:: ImportFrom(module, names, level) + + Represents ``from x import y``. ``module`` is a raw string of the 'from' name, + without any leading dots, or ``None`` for statements such as ``from . import foo``. + ``level`` is an integer holding the level of the relative import (0 means + absolute import). + + .. doctest:: + + >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) + Module( + body=[ + ImportFrom( + module='y', + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')], + level=0)], + type_ignores=[]) + + +.. class:: alias(name, asname) + + Both parameters are raw strings of the names. ``asname`` can be ``None`` if + the regular name is to be used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) + Module( + body=[ + ImportFrom( + module='foo.bar', + names=[ + alias(name='a', asname='b'), + alias(name='c')], + level=2)], + type_ignores=[]) + +Control flow +^^^^^^^^^^^^ + +.. note:: + Optional clauses such as ``else`` are stored as an empty list if they're + not present. + +.. class:: If(test, body, orelse) + + An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` + node. ``body`` and ``orelse`` each hold a list of nodes. + + ``elif`` clauses don't have a special representation in the AST, but rather + appear as extra :class:`If` nodes within the ``orelse`` section of the + previous one. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... if x: + ... ... + ... elif y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + If( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + If( + test=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: For(target, iter, body, orelse, type_comment) + + A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a + single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds + the item to be looped over, again as a single node. ``body`` and ``orelse`` + contain lists of nodes to execute. Those in ``orelse`` are executed if the + loop finishes normally, rather than via a ``break`` statement. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... for x in y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='x', ctx=Store()), + iter=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: While(test, body, orelse) + + A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` + node. + + .. doctest:: + + >> print(ast.dump(ast.parse(""" + ... while x: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + While( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: Break + Continue + + The ``break`` and ``continue`` statements. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... for a in b: + ... if a > 5: + ... break + ... else: + ... continue + ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='a', ctx=Store()), + iter=Name(id='b', ctx=Load()), + body=[ + If( + test=Compare( + left=Name(id='a', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + body=[ + Break()], + orelse=[ + Continue()])], + orelse=[])], + type_ignores=[]) + + +.. class:: Try(body, handlers, orelse, finalbody) + + ``try`` blocks. All attributes are list of nodes to execute, except for + ``handlers``, which is a list of :class:`ExceptHandler` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... try: + ... ... + ... except Exception: + ... ... + ... except OtherException as e: + ... ... + ... else: + ... ... + ... finally: + ... ... + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=Constant(value=Ellipsis))], + handlers=[ + ExceptHandler( + type=Name(id='Exception', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + ExceptHandler( + type=Name(id='OtherException', ctx=Load()), + name='e', + body=[ + Expr( + value=Constant(value=Ellipsis))])], + orelse=[ + Expr( + value=Constant(value=Ellipsis))], + finalbody=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: ExceptHandler(type, name, body) + + A single ``except`` clause. ``type`` is the exception type it will match, + typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). + ``name`` is a raw string for the name to hold the exception, or ``None`` if + the clause doesn't have ``as foo``. ``body`` is a list of nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... try: + ... a + 1 + ... except TypeError: + ... pass + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=BinOp( + left=Name(id='a', ctx=Load()), + op=Add(), + right=Constant(value=1)))], + handlers=[ + ExceptHandler( + type=Name(id='TypeError', ctx=Load()), + body=[ + Pass()])], + orelse=[], + finalbody=[])], + type_ignores=[]) + + +.. class:: With(items, body, type_comment) + + A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing + the context managers, and ``body`` is the indented block inside the context. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: withitem(context_expr, optional_vars) + + A single context manager in a ``with`` block. ``context_expr`` is the context + manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, + :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that + isn't used. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... with a as b, c as d: + ... something(b, d) + ... """), indent=4)) + Module( + body=[ + With( + items=[ + withitem( + context_expr=Name(id='a', ctx=Load()), + optional_vars=Name(id='b', ctx=Store())), + withitem( + context_expr=Name(id='c', ctx=Load()), + optional_vars=Name(id='d', ctx=Store()))], + body=[ + Expr( + value=Call( + func=Name(id='something', ctx=Load()), + args=[ + Name(id='b', ctx=Load()), + Name(id='d', ctx=Load())], + keywords=[]))])], + type_ignores=[]) + + +.. class:: Match(subject, cases) + + A ``match`` statement. ``subject`` holds the subject of the match (the object + that is being matched against the cases) and ``cases`` contains an iterable of + :class:`match_case` nodes with the different cases. + + +.. class:: match_case(context_expr, optional_vars) + + A single case pattern in a ``match`` statement. ``pattern`` contains the + match pattern that will be used to match the subject against. Notice that + the meaning of the :class:`AST` nodes in this attribute have a different + meaning than in other places, as they represent patterns to match against. + The ``guard`` attribute contains an expression that will be evaluated if + the pattern matches the subject. If the pattern matches and the ``guard`` condition + is truthy, the body of the case shall be executed. ``body`` contains a list + of nodes to execute if the guard is truthy. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] if x>0: + ... ... + ... case tuple(): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + guard=Compare( + left=Name(id='x', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=0)]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=Call( + func=Name(id='tuple', ctx=Load()), + args=[], + keywords=[]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchAs(pattern, name) + + A match "as-pattern". The as-pattern matches whatever pattern is on its + left-hand side, but also binds the value to a name. ``pattern`` contains + the match pattern that will be used to match the subject agsinst. The ``name`` + attribute contains the name that will be binded if the pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] as y: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchAs( + pattern=List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + name='y'), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: MatchOr(patterns) + + A match "or-pattern". An or-pattern matches each of its subpatterns in turn + to the subject, until one succeeds. The or-pattern is then deemed to + succeed. If none of the subpatterns succeed the or-pattern fails. The + ``patterns`` attribute contains a list of match patterns nodes that will be + matched against the subject. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] | (y): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchOr( + patterns=[ + List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + Name(id='y', ctx=Store())]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +Function and class definitions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) + + A function definition. + + * ``name`` is a raw string of the function name. + * ``args`` is a :class:`arguments` node. + * ``body`` is the list of nodes inside the function. + * ``decorator_list`` is the list of decorators to be applied, stored outermost + first (i.e. the first in the list will be applied last). + * ``returns`` is the return annotation. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: Lambda(args, body) + + ``lambda`` is a minimal function definition that can be used inside an + expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) + Module( + body=[ + Expr( + value=Lambda( + args=arguments( + posonlyargs=[], + args=[ + arg(arg='x'), + arg(arg='y')], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=Constant(value=Ellipsis)))], + type_ignores=[]) + + +.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) + + The arguments for a function. + + * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. + * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the + ``*args, **kwargs`` parameters. + * ``kw_defaults`` is a list of default values for keyword-only arguments. If + one is ``None``, the corresponding argument is required. + * ``defaults`` is a list of default values for arguments that can be passed + positionally. If there are fewer defaults, they correspond to the last n + arguments. + + +.. class:: arg(arg, annotation, type_comment) + + A single argument in a list. ``arg`` is a raw string of the argument + name, ``annotation`` is its annotation, such as a :class:`Str` or + :class:`Name` node. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': + ... pass + ... """), indent=4)) + Module( + body=[ + FunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[ + arg( + arg='a', + annotation=Constant(value='annotation')), + arg(arg='b'), + arg(arg='c')], + vararg=arg(arg='d'), + kwonlyargs=[ + arg(arg='e'), + arg(arg='f')], + kw_defaults=[ + None, + Constant(value=3)], + kwarg=arg(arg='g'), + defaults=[ + Constant(value=1), + Constant(value=2)]), + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())], + returns=Constant(value='return annotation'))], + type_ignores=[]) + + +.. class:: Return(value) + + A ``return`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('return 4'), indent=4)) + Module( + body=[ + Return( + value=Constant(value=4))], + type_ignores=[]) + + +.. class:: Yield(value) + YieldFrom(value) + + A ``yield`` or ``yield from`` expression. Because these are expressions, they + must be wrapped in a :class:`Expr` node if the value sent back is not used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('yield x'), indent=4)) + Module( + body=[ + Expr( + value=Yield( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('yield from x'), indent=4)) + Module( + body=[ + Expr( + value=YieldFrom( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + +.. class:: Global(names) + Nonlocal(names) + + ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. + + .. doctest:: + + >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) + Module( + body=[ + Global( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) + Module( + body=[ + Nonlocal( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + +.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) + + A class definition. + + * ``name`` is a raw string for the class name + * ``bases`` is a list of nodes for explicitly specified base classes. + * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. + Other keywords will be passed to the metaclass, as per `PEP-3115 + `_. + * ``starargs`` and ``kwargs`` are each a single node, as in a function call. + starargs will be expanded to join the list of base classes, and kwargs will + be passed to the metaclass. + * ``body`` is a list of nodes representing the code within the class + definition. + * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... class Foo(base1, base2, metaclass=meta): + ... pass + ... """), indent=4)) + Module( + body=[ + ClassDef( + name='Foo', + bases=[ + Name(id='base1', ctx=Load()), + Name(id='base2', ctx=Load())], + keywords=[ + keyword( + arg='metaclass', + value=Name(id='meta', ctx=Load()))], + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())])], + type_ignores=[]) + +Async and await +^^^^^^^^^^^^^^^ + +.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) + + An ``async def`` function definition. Has the same fields as + :class:`FunctionDef`. + + +.. class:: Await(value) + + An ``await`` expression. ``value`` is what it waits for. + Only valid in the body of an :class:`AsyncFunctionDef`. + +.. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[])], + type_ignores=[]) + + +.. class:: AsyncFor(target, iter, body, orelse, type_comment) + AsyncWith(items, body, type_comment) + + ``async for`` loops and ``async with`` context managers. They have the same + fields as :class:`For` and :class:`With`, respectively. Only valid in the + body of an :class:`AsyncFunctionDef`. + +.. note:: + When a string is parsed by :func:`ast.parse`, operator nodes (subclasses + of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, + :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree + will be singletons. Changes to one will be reflected in all other + occurrences of the same value (e.g. :class:`ast.Add`). + + +:mod:`ast` Helpers +------------------ + +Apart from the node classes, the :mod:`ast` module defines these utility functions +and classes for traversing abstract syntax trees: + +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) + + Parse the source into an AST node. Equivalent to ``compile(source, + filename, mode, ast.PyCF_ONLY_AST)``. + + If ``type_comments=True`` is given, the parser is modified to check + and return type comments as specified by :pep:`484` and :pep:`526`. + This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the + flags passed to :func:`compile()`. This will report syntax errors + for misplaced type comments. Without this flag, type comments will + be ignored, and the ``type_comment`` field on selected AST nodes + will always be ``None``. In addition, the locations of ``# type: + ignore`` comments will be returned as the ``type_ignores`` + attribute of :class:`Module` (otherwise it is always an empty list). + + In addition, if ``mode`` is ``'func_type'``, the input syntax is + modified to correspond to :pep:`484` "signature type comments", + e.g. ``(str, int) -> List[str]``. + + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. + + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + + .. versionchanged:: 3.8 + Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. + + +.. function:: unparse(ast_obj) + + Unparse an :class:`ast.AST` object and generate a string with code + that would produce an equivalent :class:`ast.AST` object if parsed + back with :func:`ast.parse`. + + .. warning:: + The produced code string will not necessarily be equal to the original + code that generated the :class:`ast.AST` object (without any compiler + optimizations, such as constant tuples/frozensets). + + .. warning:: + Trying to unparse a highly complex expression would result with + :exc:`RecursionError`. + + .. versionadded:: 3.9 + + +.. function:: literal_eval(node_or_string) + + Safely evaluate an expression node or a string containing a Python literal or + container display. The string or node provided may only consist of the + following Python literal structures: strings, bytes, numbers, tuples, lists, + dicts, sets, booleans, ``None`` and ``Ellipsis``. + + This can be used for safely evaluating strings containing Python values from + untrusted sources without the need to parse the values oneself. It is not + capable of evaluating arbitrarily complex expressions, for example involving + operators or indexing. + + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + + It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, + :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed + input. + + .. versionchanged:: 3.2 + Now allows bytes and set literals. + + .. versionchanged:: 3.9 + Now supports creating empty sets with ``'set()'``. + + .. versionchanged:: 3.10 + For string inputs, leading spaces and tabs are now stripped. + + +.. function:: get_docstring(node, clean=True) + + Return the docstring of the given *node* (which must be a + :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, + or :class:`Module` node), or ``None`` if it has no docstring. + If *clean* is true, clean up the docstring's indentation with + :func:`inspect.cleandoc`. + + .. versionchanged:: 3.5 + :class:`AsyncFunctionDef` is now supported. + + +.. function:: get_source_segment(source, node, *, padded=False) + + Get source code segment of the *source* that generated *node*. + If some location information (:attr:`lineno`, :attr:`end_lineno`, + :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. + + If *padded* is ``True``, the first line of a multi-line statement will + be padded with spaces to match its original position. + + .. versionadded:: 3.8 + + +.. function:: fix_missing_locations(node) + + When you compile a node tree with :func:`compile`, the compiler expects + :attr:`lineno` and :attr:`col_offset` attributes for every node that supports + them. This is rather tedious to fill in for generated nodes, so this helper + adds these attributes recursively where not already set, by setting them to + the values of the parent node. It works recursively starting at *node*. + + +.. function:: increment_lineno(node, n=1) + + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file. + + +.. function:: copy_location(new_node, old_node) + + Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, + and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, + and return *new_node*. + + +.. function:: iter_fields(node) + + Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` + that is present on *node*. + + +.. function:: iter_child_nodes(node) + + Yield all direct child nodes of *node*, that is, all fields that are nodes + and all items of fields that are lists of nodes. + + +.. function:: walk(node) + + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you only + want to modify nodes in place and don't care about the context. + + +.. class:: NodeVisitor() + + A node visitor base class that walks the abstract syntax tree and calls a + visitor function for every node found. This function may return a value + which is forwarded by the :meth:`visit` method. + + This class is meant to be subclassed, with the subclass adding visitor + methods. + + .. method:: visit(node) + + Visit a node. The default implementation calls the method called + :samp:`self.visit_{classname}` where *classname* is the name of the node + class, or :meth:`generic_visit` if that method doesn't exist. + + .. method:: generic_visit(node) + + This visitor calls :meth:`visit` on all children of the node. + + Note that child nodes of nodes that have a custom visitor method won't be + visited unless the visitor calls :meth:`generic_visit` or visits them + itself. + + Don't use the :class:`NodeVisitor` if you want to apply changes to nodes + during traversal. For this a special visitor exists + (:class:`NodeTransformer`) that allows modifications. + + .. deprecated:: 3.8 + + Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, + :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + now and will not be called in future Python versions. Add the + :meth:`visit_Constant` method to handle all constant nodes. + + +.. class:: NodeTransformer() + + A :class:`NodeVisitor` subclass that walks the abstract syntax tree and + allows modification of nodes. + + The :class:`NodeTransformer` will walk the AST and use the return value of + the visitor methods to replace or remove the old node. If the return value + of the visitor method is ``None``, the node will be removed from its + location, otherwise it is replaced with the return value. The return value + may be the original node in which case no replacement takes place. + + Here is an example transformer that rewrites all occurrences of name lookups + (``foo``) to ``data['foo']``:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return Subscript( + value=Name(id='data', ctx=Load()), + slice=Constant(value=node.id), + ctx=node.ctx + ) + + Keep in mind that if the node you're operating on has child nodes you must + either transform the child nodes yourself or call the :meth:`generic_visit` + method for the node first. + + For nodes that were part of a collection of statements (that applies to all + statement nodes), the visitor may also return a list of nodes rather than + just a single node. + + If :class:`NodeTransformer` introduces new nodes (that weren't part of + original tree) without giving them location information (such as + :attr:`lineno`), :func:`fix_missing_locations` should be called with + the new sub-tree to recalculate the location information:: + + tree = ast.parse('foo', mode='eval') + new_tree = fix_missing_locations(RewriteName().visit(tree)) + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + + +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) + + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. If *annotate_fields* is true (by default), + the returned string will show the names and the values for fields. + If *annotate_fields* is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line + numbers and column offsets are not dumped by default. If this is wanted, + *include_attributes* can be set to true. + + If *indent* is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. An indent level + of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) + selects the single line representation. Using a positive integer indent + indents that many spaces per level. If *indent* is a string (such as ``"\t"``), + that string is used to indent each level. + + .. versionchanged:: 3.9 + Added the *indent* option. + + +.. _ast-compiler-flags: + +Compiler Flags +-------------- + +The following flags may be passed to :func:`compile` in order to change +effects on the compilation of a program: + +.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT + + Enables support for top-level ``await``, ``async for``, ``async with`` + and async comprehensions. + + .. versionadded:: 3.8 + +.. data:: PyCF_ONLY_AST + + Generates and returns an abstract syntax tree instead of returning a + compiled code object. + +.. data:: PyCF_TYPE_COMMENTS + + Enables support for :pep:`484` and :pep:`526` style type comments + (``# type: ``, ``# type: ignore ``). + + .. versionadded:: 3.8 + + +.. _ast-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 3.9 + +The :mod:`ast` module can be executed as a script from the command line. +It is as simple as: + +.. code-block:: sh + + python -m ast [-m ] [-a] [infile] + +The following options are accepted: + +.. program:: ast + +.. cmdoption:: -h, --help + + Show the help message and exit. + +.. cmdoption:: -m + --mode + + Specify what kind of code must be compiled, like the *mode* argument + in :func:`parse`. + +.. cmdoption:: --no-type-comments + + Don't parse type comments. + +.. cmdoption:: -a, --include-attributes + + Include attributes such as line numbers and column offsets. + +.. cmdoption:: -i + --indent + + Indentation of nodes in AST (number of spaces). + +If :file:`infile` is specified its contents are parsed to AST and dumped +to stdout. Otherwise, the content is read from stdin. + + +.. seealso:: + + `Green Tree Snakes `_, an external + documentation resource, has good details on working with Python ASTs. + + `ASTTokens `_ + annotates Python ASTs with the positions of tokens and text in the source + code that generated them. This is helpful for tools that make source code + transformations. + + `leoAst.py `_ unifies the + token-based and parse-tree-based views of python programs by inserting + two-way links between tokens and ast nodes. + + `LibCST `_ parses code as a Concrete Syntax + Tree that looks like an ast tree and keeps all formatting details. It's + useful for building automated refactoring (codemod) applications and + linters. + + `Parso `_ is a Python parser that supports + error recovery and round-trip parsing for different Python versions (in + multiple Python versions). Parso is also able to list multiple syntax errors + in your python file. diff --git a/Doc/library/asyncio-api-index.rst.bak b/Doc/library/asyncio-api-index.rst.bak new file mode 100644 index 000000000000000..047e5bbc58ccade --- /dev/null +++ b/Doc/library/asyncio-api-index.rst.bak @@ -0,0 +1,221 @@ +.. currentmodule:: asyncio + + +==================== +High-level API Index +==================== + +This page lists all high-level async/await enabled asyncio APIs. + + +Tasks +===== + +Utilities to run asyncio programs, create Tasks, and +await on multiple things with timeouts. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :func:`run` + - Create event loop, run a coroutine, close the loop. + + * - :func:`create_task` + - Start an asyncio Task. + + * - ``await`` :func:`sleep` + - Sleep for a number of seconds. + + * - ``await`` :func:`gather` + - Schedule and wait for things concurrently. + + * - ``await`` :func:`wait_for` + - Run with a timeout. + + * - ``await`` :func:`shield` + - Shield from cancellation. + + * - ``await`` :func:`wait` + - Monitor for completion. + + * - :func:`current_task` + - Return the current Task. + + * - :func:`all_tasks` + - Return all tasks for an event loop. + + * - :class:`Task` + - Task object. + + * - :func:`to_thread` + - Asychronously run a function in a separate OS thread. + + * - :func:`run_coroutine_threadsafe` + - Schedule a coroutine from another OS thread. + + * - ``for in`` :func:`as_completed` + - Monitor for completion with a ``for`` loop. + + +.. rubric:: Examples + +* :ref:`Using asyncio.gather() to run things in parallel + `. + +* :ref:`Using asyncio.wait_for() to enforce a timeout + `. + +* :ref:`Cancellation `. + +* :ref:`Using asyncio.sleep() `. + +* See also the main :ref:`Tasks documentation page `. + + +Queues +====== + +Queues should be used to distribute work amongst multiple asyncio Tasks, +implement connection pools, and pub/sub patterns. + + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Queue` + - A FIFO queue. + + * - :class:`PriorityQueue` + - A priority queue. + + * - :class:`LifoQueue` + - A LIFO queue. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Queue to distribute workload between several + Tasks `. + +* See also the :ref:`Queues documentation page `. + + +Subprocesses +============ + +Utilities to spawn subprocesses and run shell commands. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`create_subprocess_exec` + - Create a subprocess. + + * - ``await`` :func:`create_subprocess_shell` + - Run a shell command. + + +.. rubric:: Examples + +* :ref:`Executing a shell command `. + +* See also the :ref:`subprocess APIs ` + documentation. + + +Streams +======= + +High-level APIs to work with network IO. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`open_connection` + - Establish a TCP connection. + + * - ``await`` :func:`open_unix_connection` + - Establish a Unix socket connection. + + * - ``await`` :func:`start_server` + - Start a TCP server. + + * - ``await`` :func:`start_unix_server` + - Start a Unix socket server. + + * - :class:`StreamReader` + - High-level async/await object to receive network data. + + * - :class:`StreamWriter` + - High-level async/await object to send network data. + + +.. rubric:: Examples + +* :ref:`Example TCP client `. + +* See also the :ref:`streams APIs ` + documentation. + + +Synchronization +=============== + +Threading-like synchronization primitives that can be used in Tasks. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Lock` + - A mutex lock. + + * - :class:`Event` + - An event object. + + * - :class:`Condition` + - A condition object. + + * - :class:`Semaphore` + - A semaphore. + + * - :class:`BoundedSemaphore` + - A bounded semaphore. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Event `. + +* See also the documentation of asyncio + :ref:`synchronization primitives `. + + +Exceptions +========== + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + + * - :exc:`asyncio.TimeoutError` + - Raised on timeout by functions like :func:`wait_for`. + Keep in mind that ``asyncio.TimeoutError`` is **unrelated** + to the built-in :exc:`TimeoutError` exception. + + * - :exc:`asyncio.CancelledError` + - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + + +.. rubric:: Examples + +* :ref:`Handling CancelledError to run code on cancellation request + `. + +* See also the full list of + :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/collections.rst.bak b/Doc/library/collections.rst.bak new file mode 100644 index 000000000000000..aa0acfaae45a5f0 --- /dev/null +++ b/Doc/library/collections.rst.bak @@ -0,0 +1,1278 @@ +:mod:`collections` --- Container datatypes +========================================== + +.. module:: collections + :synopsis: Container datatypes + +.. moduleauthor:: Raymond Hettinger +.. sectionauthor:: Raymond Hettinger + +**Source code:** :source:`Lib/collections/__init__.py` + +.. testsetup:: * + + from collections import * + import itertools + __name__ = '' + +-------------- + +This module implements specialized container datatypes providing alternatives to +Python's general purpose built-in containers, :class:`dict`, :class:`list`, +:class:`set`, and :class:`tuple`. + +===================== ==================================================================== +:func:`namedtuple` factory function for creating tuple subclasses with named fields +:class:`deque` list-like container with fast appends and pops on either end +:class:`ChainMap` dict-like class for creating a single view of multiple mappings +:class:`Counter` dict subclass for counting hashable objects +:class:`OrderedDict` dict subclass that remembers the order entries were added +:class:`defaultdict` dict subclass that calls a factory function to supply missing values +:class:`UserDict` wrapper around dictionary objects for easier dict subclassing +:class:`UserList` wrapper around list objects for easier list subclassing +:class:`UserString` wrapper around string objects for easier string subclassing +===================== ==================================================================== + + +:class:`ChainMap` objects +------------------------- + +.. versionadded:: 3.3 + +A :class:`ChainMap` class is provided for quickly linking a number of mappings +so they can be treated as a single unit. It is often much faster than creating +a new dictionary and running multiple :meth:`~dict.update` calls. + +The class can be used to simulate nested scopes and is useful in templating. + +.. class:: ChainMap(*maps) + + A :class:`ChainMap` groups multiple dicts or other mappings together to + create a single, updateable view. If no *maps* are specified, a single empty + dictionary is provided so that a new chain always has at least one mapping. + + The underlying mappings are stored in a list. That list is public and can + be accessed or updated using the *maps* attribute. There is no other state. + + Lookups search the underlying mappings successively until a key is found. In + contrast, writes, updates, and deletions only operate on the first mapping. + + A :class:`ChainMap` incorporates the underlying mappings by reference. So, if + one of the underlying mappings gets updated, those changes will be reflected + in :class:`ChainMap`. + + All of the usual dictionary methods are supported. In addition, there is a + *maps* attribute, a method for creating new subcontexts, and a property for + accessing all but the first mapping: + + .. attribute:: maps + + A user updateable list of mappings. The list is ordered from + first-searched to last-searched. It is the only stored state and can + be modified to change which mappings are searched. The list should + always contain at least one mapping. + + .. method:: new_child(m=None) + + Returns a new :class:`ChainMap` containing a new map followed by + all of the maps in the current instance. If ``m`` is specified, + it becomes the new map at the front of the list of mappings; if not + specified, an empty dict is used, so that a call to ``d.new_child()`` + is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for + creating subcontexts that can be updated without altering values in any + of the parent mappings. + + .. versionchanged:: 3.4 + The optional ``m`` parameter was added. + + .. attribute:: parents + + Property returning a new :class:`ChainMap` containing all of the maps in + the current instance except the first one. This is useful for skipping + the first map in the search. Use cases are similar to those for the + :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in + :func:`super` function. A reference to ``d.parents`` is equivalent to: + ``ChainMap(*d.maps[1:])``. + + Note, the iteration order of a :class:`ChainMap()` is determined by + scanning the mappings last to first:: + + >>> baseline = {'music': 'bach', 'art': 'rembrandt'} + >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} + >>> list(ChainMap(adjustments, baseline)) + ['music', 'art', 'opera'] + + This gives the same ordering as a series of :meth:`dict.update` calls + starting with the last mapping:: + + >>> combined = baseline.copy() + >>> combined.update(adjustments) + >>> list(combined) + ['music', 'art', 'opera'] + + .. versionchanged:: 3.9 + Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. + +.. seealso:: + + * The `MultiContext class + `_ + in the Enthought `CodeTools package + `_ has options to support + writing to any mapping in the chain. + + * Django's `Context class + `_ + for templating is a read-only chain of mappings. It also features + pushing and popping of contexts similar to the + :meth:`~collections.ChainMap.new_child` method and the + :attr:`~collections.ChainMap.parents` property. + + * The `Nested Contexts recipe + `_ has options to control + whether writes and other mutations apply only to the first mapping or to + any mapping in the chain. + + * A `greatly simplified read-only version of Chainmap + `_. + + +:class:`ChainMap` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section shows various approaches to working with chained maps. + + +Example of simulating Python's internal lookup chain:: + + import builtins + pylookup = ChainMap(locals(), globals(), vars(builtins)) + +Example of letting user specified command-line arguments take precedence over +environment variables which in turn take precedence over default values:: + + import os, argparse + + defaults = {'color': 'red', 'user': 'guest'} + + parser = argparse.ArgumentParser() + parser.add_argument('-u', '--user') + parser.add_argument('-c', '--color') + namespace = parser.parse_args() + command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} + + combined = ChainMap(command_line_args, os.environ, defaults) + print(combined['color']) + print(combined['user']) + +Example patterns for using the :class:`ChainMap` class to simulate nested +contexts:: + + c = ChainMap() # Create root context + d = c.new_child() # Create nested child context + e = c.new_child() # Child of c, independent from d + e.maps[0] # Current context dictionary -- like Python's locals() + e.maps[-1] # Root context -- like Python's globals() + e.parents # Enclosing context chain -- like Python's nonlocals + + d['x'] = 1 # Set value in current context + d['x'] # Get first key in the chain of contexts + del d['x'] # Delete from current context + list(d) # All nested values + k in d # Check all nested values + len(d) # Number of nested values + d.items() # All nested items + dict(d) # Flatten into a regular dictionary + +The :class:`ChainMap` class only makes updates (writes and deletions) to the +first mapping in the chain while lookups will search the full chain. However, +if deep writes and deletions are desired, it is easy to make a subclass that +updates keys found deeper in the chain:: + + class DeepChainMap(ChainMap): + 'Variant of ChainMap that allows direct updates to inner scopes' + + def __setitem__(self, key, value): + for mapping in self.maps: + if key in mapping: + mapping[key] = value + return + self.maps[0][key] = value + + def __delitem__(self, key): + for mapping in self.maps: + if key in mapping: + del mapping[key] + return + raise KeyError(key) + + >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) + >>> d['lion'] = 'orange' # update an existing key two levels down + >>> d['snake'] = 'red' # new keys get added to the topmost dict + >>> del d['elephant'] # remove an existing key one level down + >>> d # display result + DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) + + +:class:`Counter` objects +------------------------ + +A counter tool is provided to support convenient and rapid tallies. +For example:: + + >>> # Tally occurrences of words in a list + >>> cnt = Counter() + >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: + ... cnt[word] += 1 + >>> cnt + Counter({'blue': 3, 'red': 2, 'green': 1}) + + >>> # Find the ten most common words in Hamlet + >>> import re + >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) + >>> Counter(words).most_common(10) + [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), + ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] + +.. class:: Counter([iterable-or-mapping]) + + A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. + It is a collection where elements are stored as dictionary keys + and their counts are stored as dictionary values. Counts are allowed to be + any integer value including zero or negative counts. The :class:`Counter` + class is similar to bags or multisets in other languages. + + Elements are counted from an *iterable* or initialized from another + *mapping* (or counter): + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping + >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args + + Counter objects have a dictionary interface except that they return a zero + count for missing items instead of raising a :exc:`KeyError`: + + >>> c = Counter(['eggs', 'ham']) + >>> c['bacon'] # count of a missing element is zero + 0 + + Setting a count to zero does not remove an element from a counter. + Use ``del`` to remove it entirely: + + >>> c['sausage'] = 0 # counter entry with a zero count + >>> del c['sausage'] # del actually removes the entry + + .. versionadded:: 3.1 + + .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` + Inherited the capability to remember insertion order. Math operations + on *Counter* objects also preserve order. Results are ordered + according to when an element is first encountered in the left operand + and then by the order encountered in the right operand. + + Counter objects support three methods beyond those available for all + dictionaries: + + .. method:: elements() + + Return an iterator over elements repeating each as many times as its + count. Elements are returned in the order first encountered. If an + element's count is less than one, :meth:`elements` will ignore it. + + >>> c = Counter(a=4, b=2, c=0, d=-2) + >>> sorted(c.elements()) + ['a', 'a', 'a', 'a', 'b', 'b'] + + .. method:: most_common([n]) + + Return a list of the *n* most common elements and their counts from the + most common to the least. If *n* is omitted or ``None``, + :meth:`most_common` returns *all* elements in the counter. + Elements with equal counts are ordered in the order first encountered: + + >>> Counter('abracadabra').most_common(3) + [('a', 5), ('b', 2), ('r', 2)] + + .. method:: subtract([iterable-or-mapping]) + + Elements are subtracted from an *iterable* or from another *mapping* + (or counter). Like :meth:`dict.update` but subtracts counts instead + of replacing them. Both inputs and outputs may be zero or negative. + + >>> c = Counter(a=4, b=2, c=0, d=-2) + >>> d = Counter(a=1, b=2, c=3, d=4) + >>> c.subtract(d) + >>> c + Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) + + .. versionadded:: 3.2 + + The usual dictionary methods are available for :class:`Counter` objects + except for two which work differently for counters. + + .. method:: fromkeys(iterable) + + This class method is not implemented for :class:`Counter` objects. + + .. method:: update([iterable-or-mapping]) + + Elements are counted from an *iterable* or added-in from another + *mapping* (or counter). Like :meth:`dict.update` but adds counts + instead of replacing them. Also, the *iterable* is expected to be a + sequence of elements, not a sequence of ``(key, value)`` pairs. + +Counters support rich comparison operators for equality, subset, and +superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. +All of those tests treat missing elements as having zero counts so that +``Counter(a=1) == Counter(a=1, b=0)`` returns true. + +.. versionadded:: 3.10 + Rich comparison operations we were added + +.. versionchanged:: 3.10 + In equality tests, missing elements are treated as having zero counts. + Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered + distinct. + +Common patterns for working with :class:`Counter` objects:: + + sum(c.values()) # total of all counts + c.clear() # reset all counts + list(c) # list unique elements + set(c) # convert to a set + dict(c) # convert to a regular dictionary + c.items() # convert to a list of (elem, cnt) pairs + Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs + c.most_common()[:-n-1:-1] # n least common elements + +c # remove zero and negative counts + +Several mathematical operations are provided for combining :class:`Counter` +objects to produce multisets (counters that have counts greater than zero). +Addition and subtraction combine counters by adding or subtracting the counts +of corresponding elements. Intersection and union return the minimum and +maximum of corresponding counts. Each operation can accept inputs with signed +counts, but the output will exclude results with counts of zero or less. + + >>> c = Counter(a=3, b=1) + >>> d = Counter(a=1, b=2) + >>> c + d # add two counters together: c[x] + d[x] + Counter({'a': 4, 'b': 3}) + >>> c - d # subtract (keeping only positive counts) + Counter({'a': 2}) + >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP + Counter({'a': 1, 'b': 1}) + >>> c | d # union: max(c[x], d[x]) + Counter({'a': 3, 'b': 2}) + +Unary addition and subtraction are shortcuts for adding an empty counter +or subtracting from an empty counter. + + >>> c = Counter(a=2, b=-4) + >>> +c + Counter({'a': 2}) + >>> -c + Counter({'b': 4}) + +.. versionadded:: 3.3 + Added support for unary plus, unary minus, and in-place multiset operations. + +.. note:: + + Counters were primarily designed to work with positive integers to represent + running counts; however, care was taken to not unnecessarily preclude use + cases needing other types or negative values. To help with those use cases, + this section documents the minimum range and type restrictions. + + * The :class:`Counter` class itself is a dictionary subclass with no + restrictions on its keys and values. The values are intended to be numbers + representing counts, but you *could* store anything in the value field. + + * The :meth:`~Counter.most_common` method requires only that the values be orderable. + + * For in-place operations such as ``c[key] += 1``, the value type need only + support addition and subtraction. So fractions, floats, and decimals would + work and negative values are supported. The same is also true for + :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values + for both inputs and outputs. + + * The multiset methods are designed only for use cases with positive values. + The inputs may be negative or zero, but only outputs with positive values + are created. There are no type restrictions, but the value type needs to + support addition, subtraction, and comparison. + + * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and + negative counts. + +.. seealso:: + + * `Bag class `_ + in Smalltalk. + + * Wikipedia entry for `Multisets `_. + + * `C++ multisets `_ + tutorial with examples. + + * For mathematical operations on multisets and their use cases, see + *Knuth, Donald. The Art of Computer Programming Volume II, + Section 4.6.3, Exercise 19*. + + * To enumerate all distinct multisets of a given size over a given set of + elements, see :func:`itertools.combinations_with_replacement`:: + + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC + + +:class:`deque` objects +---------------------- + +.. class:: deque([iterable, [maxlen]]) + + Returns a new deque object initialized left-to-right (using :meth:`append`) with + data from *iterable*. If *iterable* is not specified, the new deque is empty. + + Deques are a generalization of stacks and queues (the name is pronounced "deck" + and is short for "double-ended queue"). Deques support thread-safe, memory + efficient appends and pops from either side of the deque with approximately the + same O(1) performance in either direction. + + Though :class:`list` objects support similar operations, they are optimized for + fast fixed-length operations and incur O(n) memory movement costs for + ``pop(0)`` and ``insert(0, v)`` operations which change both the size and + position of the underlying data representation. + + + If *maxlen* is not specified or is ``None``, deques may grow to an + arbitrary length. Otherwise, the deque is bounded to the specified maximum + length. Once a bounded length deque is full, when new items are added, a + corresponding number of items are discarded from the opposite end. Bounded + length deques provide functionality similar to the ``tail`` filter in + Unix. They are also useful for tracking transactions and other pools of data + where only the most recent activity is of interest. + + + Deque objects support the following methods: + + .. method:: append(x) + + Add *x* to the right side of the deque. + + + .. method:: appendleft(x) + + Add *x* to the left side of the deque. + + + .. method:: clear() + + Remove all elements from the deque leaving it with length 0. + + + .. method:: copy() + + Create a shallow copy of the deque. + + .. versionadded:: 3.5 + + + .. method:: count(x) + + Count the number of deque elements equal to *x*. + + .. versionadded:: 3.2 + + + .. method:: extend(iterable) + + Extend the right side of the deque by appending elements from the iterable + argument. + + + .. method:: extendleft(iterable) + + Extend the left side of the deque by appending elements from *iterable*. + Note, the series of left appends results in reversing the order of + elements in the iterable argument. + + + .. method:: index(x[, start[, stop]]) + + Return the position of *x* in the deque (at or after index *start* + and before index *stop*). Returns the first match or raises + :exc:`ValueError` if not found. + + .. versionadded:: 3.5 + + + .. method:: insert(i, x) + + Insert *x* into the deque at position *i*. + + If the insertion would cause a bounded deque to grow beyond *maxlen*, + an :exc:`IndexError` is raised. + + .. versionadded:: 3.5 + + + .. method:: pop() + + Remove and return an element from the right side of the deque. If no + elements are present, raises an :exc:`IndexError`. + + + .. method:: popleft() + + Remove and return an element from the left side of the deque. If no + elements are present, raises an :exc:`IndexError`. + + + .. method:: remove(value) + + Remove the first occurrence of *value*. If not found, raises a + :exc:`ValueError`. + + + .. method:: reverse() + + Reverse the elements of the deque in-place and then return ``None``. + + .. versionadded:: 3.2 + + + .. method:: rotate(n=1) + + Rotate the deque *n* steps to the right. If *n* is negative, rotate + to the left. + + When the deque is not empty, rotating one step to the right is equivalent + to ``d.appendleft(d.pop())``, and rotating one step to the left is + equivalent to ``d.append(d.popleft())``. + + + Deque objects also provide one read-only attribute: + + .. attribute:: maxlen + + Maximum size of a deque or ``None`` if unbounded. + + .. versionadded:: 3.1 + + +In addition to the above, deques support iteration, pickling, ``len(d)``, +``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with +the :keyword:`in` operator, and subscript references such as ``d[0]`` to access +the first element. Indexed access is O(1) at both ends but slows to O(n) in +the middle. For fast random access, use lists instead. + +Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, +and ``__imul__()``. + +Example: + +.. doctest:: + + >>> from collections import deque + >>> d = deque('ghi') # make a new deque with three items + >>> for elem in d: # iterate over the deque's elements + ... print(elem.upper()) + G + H + I + + >>> d.append('j') # add a new entry to the right side + >>> d.appendleft('f') # add a new entry to the left side + >>> d # show the representation of the deque + deque(['f', 'g', 'h', 'i', 'j']) + + >>> d.pop() # return and remove the rightmost item + 'j' + >>> d.popleft() # return and remove the leftmost item + 'f' + >>> list(d) # list the contents of the deque + ['g', 'h', 'i'] + >>> d[0] # peek at leftmost item + 'g' + >>> d[-1] # peek at rightmost item + 'i' + + >>> list(reversed(d)) # list the contents of a deque in reverse + ['i', 'h', 'g'] + >>> 'h' in d # search the deque + True + >>> d.extend('jkl') # add multiple elements at once + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + >>> d.rotate(1) # right rotation + >>> d + deque(['l', 'g', 'h', 'i', 'j', 'k']) + >>> d.rotate(-1) # left rotation + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + + >>> deque(reversed(d)) # make a new deque in reverse order + deque(['l', 'k', 'j', 'i', 'h', 'g']) + >>> d.clear() # empty the deque + >>> d.pop() # cannot pop from an empty deque + Traceback (most recent call last): + File "", line 1, in -toplevel- + d.pop() + IndexError: pop from an empty deque + + >>> d.extendleft('abc') # extendleft() reverses the input order + >>> d + deque(['c', 'b', 'a']) + + +:class:`deque` Recipes +^^^^^^^^^^^^^^^^^^^^^^ + +This section shows various approaches to working with deques. + +Bounded length deques provide functionality similar to the ``tail`` filter +in Unix:: + + def tail(filename, n=10): + 'Return the last n lines of a file' + with open(filename) as f: + return deque(f, n) + +Another approach to using deques is to maintain a sequence of recently +added elements by appending to the right and popping to the left:: + + def moving_average(iterable, n=3): + # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 + # http://en.wikipedia.org/wiki/Moving_average + it = iter(iterable) + d = deque(itertools.islice(it, n-1)) + d.appendleft(0) + s = sum(d) + for elem in it: + s += elem - d.popleft() + d.append(elem) + yield s / n + +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + +The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and +deletion. For example, a pure Python implementation of ``del d[n]`` relies on +the ``rotate()`` method to position elements to be popped:: + + def delete_nth(d, n): + d.rotate(-n) + d.popleft() + d.rotate(n) + +To implement :class:`deque` slicing, use a similar approach applying +:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove +old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then +reverse the rotation. +With minor variations on that approach, it is easy to implement Forth style +stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, +``rot``, and ``roll``. + + +:class:`defaultdict` objects +---------------------------- + +.. class:: defaultdict([default_factory[, ...]]) + + Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the + built-in :class:`dict` class. It overrides one method and adds one writable + instance variable. The remaining functionality is the same as for the + :class:`dict` class and is not documented here. + + The first argument provides the initial value for the :attr:`default_factory` + attribute; it defaults to ``None``. All remaining arguments are treated the same + as if they were passed to the :class:`dict` constructor, including keyword + arguments. + + + :class:`defaultdict` objects support the following method in addition to the + standard :class:`dict` operations: + + .. method:: __missing__(key) + + If the :attr:`default_factory` attribute is ``None``, this raises a + :exc:`KeyError` exception with the *key* as argument. + + If :attr:`default_factory` is not ``None``, it is called without arguments + to provide a default value for the given *key*, this value is inserted in + the dictionary for the *key*, and returned. + + If calling :attr:`default_factory` raises an exception this exception is + propagated unchanged. + + This method is called by the :meth:`__getitem__` method of the + :class:`dict` class when the requested key is not found; whatever it + returns or raises is then returned or raised by :meth:`__getitem__`. + + Note that :meth:`__missing__` is *not* called for any operations besides + :meth:`__getitem__`. This means that :meth:`get` will, like normal + dictionaries, return ``None`` as a default rather than using + :attr:`default_factory`. + + + :class:`defaultdict` objects support the following instance variable: + + + .. attribute:: default_factory + + This attribute is used by the :meth:`__missing__` method; it is + initialized from the first argument to the constructor, if present, or to + ``None``, if absent. + + .. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in + :pep:`584`. + + +:class:`defaultdict` Examples +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a +sequence of key-value pairs into a dictionary of lists: + + >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] + >>> d = defaultdict(list) + >>> for k, v in s: + ... d[k].append(v) + ... + >>> sorted(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + +When each key is encountered for the first time, it is not already in the +mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` +function which returns an empty :class:`list`. The :meth:`list.append` +operation then attaches the value to the new list. When keys are encountered +again, the look-up proceeds normally (returning the list for that key) and the +:meth:`list.append` operation adds another value to the list. This technique is +simpler and faster than an equivalent technique using :meth:`dict.setdefault`: + + >>> d = {} + >>> for k, v in s: + ... d.setdefault(k, []).append(v) + ... + >>> sorted(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + +Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the +:class:`defaultdict` useful for counting (like a bag or multiset in other +languages): + + >>> s = 'mississippi' + >>> d = defaultdict(int) + >>> for k in s: + ... d[k] += 1 + ... + >>> sorted(d.items()) + [('i', 4), ('m', 1), ('p', 2), ('s', 4)] + +When a letter is first encountered, it is missing from the mapping, so the +:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of +zero. The increment operation then builds up the count for each letter. + +The function :func:`int` which always returns zero is just a special case of +constant functions. A faster and more flexible way to create constant functions +is to use a lambda function which can supply any constant value (not just +zero): + + >>> def constant_factory(value): + ... return lambda: value + >>> d = defaultdict(constant_factory('')) + >>> d.update(name='John', action='ran') + >>> '%(name)s %(action)s to %(object)s' % d + 'John ran to ' + +Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the +:class:`defaultdict` useful for building a dictionary of sets: + + >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] + >>> d = defaultdict(set) + >>> for k, v in s: + ... d[k].add(v) + ... + >>> sorted(d.items()) + [('blue', {2, 4}), ('red', {1, 3})] + + +:func:`namedtuple` Factory Function for Tuples with Named Fields +---------------------------------------------------------------- + +Named tuples assign meaning to each position in a tuple and allow for more readable, +self-documenting code. They can be used wherever regular tuples are used, and +they add the ability to access fields by name instead of position index. + +.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) + + Returns a new tuple subclass named *typename*. The new subclass is used to + create tuple-like objects that have fields accessible by attribute lookup as + well as being indexable and iterable. Instances of the subclass also have a + helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` + method which lists the tuple contents in a ``name=value`` format. + + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. + + Any valid Python identifier may be used for a fieldname except for names + starting with an underscore. Valid identifiers consist of letters, digits, + and underscores but do not start with a digit or underscore and cannot be + a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, + or *raise*. + + If *rename* is true, invalid fieldnames are automatically replaced + with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is + converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword + ``def`` and the duplicate fieldname ``abc``. + + *defaults* can be ``None`` or an :term:`iterable` of default values. + Since fields with a default value must come after any fields without a + default, the *defaults* are applied to the rightmost parameters. For + example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are + ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to + ``1``, and ``z`` will default to ``2``. + + If *module* is defined, the ``__module__`` attribute of the named tuple is + set to that value. + + Named tuple instances do not have per-instance dictionaries, so they are + lightweight and require no more memory than regular tuples. + + To support pickling, the named tuple class should be assigned to a variable + that matches *typename*. + + .. versionchanged:: 3.1 + Added support for *rename*. + + .. versionchanged:: 3.6 + The *verbose* and *rename* parameters became + :ref:`keyword-only arguments `. + + .. versionchanged:: 3.6 + Added the *module* parameter. + + .. versionchanged:: 3.7 + Removed the *verbose* parameter and the :attr:`_source` attribute. + + .. versionchanged:: 3.7 + Added the *defaults* parameter and the :attr:`_field_defaults` + attribute. + +.. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> # Basic example + >>> Point = namedtuple('Point', ['x', 'y']) + >>> p = Point(11, y=22) # instantiate with positional or keyword arguments + >>> p[0] + p[1] # indexable like the plain tuple (11, 22) + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessible by name + 33 + >>> p # readable __repr__ with a name=value style + Point(x=11, y=22) + +Named tuples are especially useful for assigning field names to result tuples returned +by the :mod:`csv` or :mod:`sqlite3` modules:: + + EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') + + import csv + for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): + print(emp.name, emp.title) + + import sqlite3 + conn = sqlite3.connect('/companydata') + cursor = conn.cursor() + cursor.execute('SELECT name, age, title, department, paygrade FROM employees') + for emp in map(EmployeeRecord._make, cursor.fetchall()): + print(emp.name, emp.title) + +In addition to the methods inherited from tuples, named tuples support +three additional methods and two attributes. To prevent conflicts with +field names, the method and attribute names start with an underscore. + +.. classmethod:: somenamedtuple._make(iterable) + + Class method that makes a new instance from an existing sequence or iterable. + + .. doctest:: + + >>> t = [11, 22] + >>> Point._make(t) + Point(x=11, y=22) + +.. method:: somenamedtuple._asdict() + + Return a new :class:`dict` which maps field names to their corresponding + values: + + .. doctest:: + + >>> p = Point(x=11, y=22) + >>> p._asdict() + {'x': 11, 'y': 22} + + .. versionchanged:: 3.1 + Returns an :class:`OrderedDict` instead of a regular :class:`dict`. + + .. versionchanged:: 3.8 + Returns a regular :class:`dict` instead of an :class:`OrderedDict`. + As of Python 3.7, regular dicts are guaranteed to be ordered. If the + extra features of :class:`OrderedDict` are required, the suggested + remediation is to cast the result to the desired type: + ``OrderedDict(nt._asdict())``. + +.. method:: somenamedtuple._replace(**kwargs) + + Return a new instance of the named tuple replacing specified fields with new + values:: + + >>> p = Point(x=11, y=22) + >>> p._replace(x=33) + Point(x=33, y=22) + + >>> for partnum, record in inventory.items(): + ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) + +.. attribute:: somenamedtuple._fields + + Tuple of strings listing the field names. Useful for introspection + and for creating new named tuple types from existing named tuples. + + .. doctest:: + + >>> p._fields # view the field names + ('x', 'y') + + >>> Color = namedtuple('Color', 'red green blue') + >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) + >>> Pixel(11, 22, 128, 255, 0) + Pixel(x=11, y=22, red=128, green=255, blue=0) + +.. attribute:: somenamedtuple._field_defaults + + Dictionary mapping field names to default values. + + .. doctest:: + + >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) + >>> Account._field_defaults + {'balance': 0} + >>> Account('premium') + Account(type='premium', balance=0) + +To retrieve a field whose name is stored in a string, use the :func:`getattr` +function: + + >>> getattr(p, 'x') + 11 + +To convert a dictionary to a named tuple, use the double-star-operator +(as described in :ref:`tut-unpacking-arguments`): + + >>> d = {'x': 11, 'y': 22} + >>> Point(**d) + Point(x=11, y=22) + +Since a named tuple is a regular Python class, it is easy to add or change +functionality with a subclass. Here is how to add a calculated field and +a fixed-width print format: + +.. doctest:: + + >>> class Point(namedtuple('Point', ['x', 'y'])): + ... __slots__ = () + ... @property + ... def hypot(self): + ... return (self.x ** 2 + self.y ** 2) ** 0.5 + ... def __str__(self): + ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) + + >>> for p in Point(3, 4), Point(14, 5/7): + ... print(p) + Point: x= 3.000 y= 4.000 hypot= 5.000 + Point: x=14.000 y= 0.714 hypot=14.018 + +The subclass shown above sets ``__slots__`` to an empty tuple. This helps +keep memory requirements low by preventing the creation of instance dictionaries. + +Subclassing is not useful for adding new, stored fields. Instead, simply +create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: + + >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) + +Docstrings can be customized by making direct assignments to the ``__doc__`` +fields: + + >>> Book = namedtuple('Book', ['id', 'title', 'authors']) + >>> Book.__doc__ += ': Hardcover book in active collection' + >>> Book.id.__doc__ = '13-digit ISBN' + >>> Book.title.__doc__ = 'Title of first printing' + >>> Book.authors.__doc__ = 'List of authors sorted by last name' + +.. versionchanged:: 3.5 + Property docstrings became writeable. + +.. seealso:: + + * See :class:`typing.NamedTuple` for a way to add type hints for named + tuples. It also provides an elegant notation using the :keyword:`class` + keyword:: + + class Component(NamedTuple): + part_number: int + weight: float + description: Optional[str] = None + + * See :meth:`types.SimpleNamespace` for a mutable namespace based on an + underlying dictionary instead of a tuple. + + * The :mod:`dataclasses` module provides a decorator and functions for + automatically adding generated special methods to user-defined classes. + + +:class:`OrderedDict` objects +---------------------------- + +Ordered dictionaries are just like regular dictionaries but have some extra +capabilities relating to ordering operations. They have become less +important now that the built-in :class:`dict` class gained the ability +to remember insertion order (this new behavior became guaranteed in +Python 3.7). + +Some differences from :class:`dict` still remain: + +* The regular :class:`dict` was designed to be very good at mapping + operations. Tracking insertion order was secondary. + +* The :class:`OrderedDict` was designed to be good at reordering operations. + Space efficiency, iteration speed, and the performance of update + operations were secondary. + +* Algorithmically, :class:`OrderedDict` can handle frequent reordering + operations better than :class:`dict`. This makes it suitable for tracking + recent accesses (for example in an `LRU cache + `_). + +* The equality operation for :class:`OrderedDict` checks for matching order. + +* The :meth:`popitem` method of :class:`OrderedDict` has a different + signature. It accepts an optional argument to specify which item is popped. + +* :class:`OrderedDict` has a :meth:`move_to_end` method to + efficiently reposition an element to an endpoint. + +* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. + + +.. class:: OrderedDict([items]) + + Return an instance of a :class:`dict` subclass that has methods + specialized for rearranging dictionary order. + + .. versionadded:: 3.1 + + .. method:: popitem(last=True) + + The :meth:`popitem` method for ordered dictionaries returns and removes a + (key, value) pair. The pairs are returned in + :abbr:`LIFO (last-in, first-out)` order if *last* is true + or :abbr:`FIFO (first-in, first-out)` order if false. + + .. method:: move_to_end(key, last=True) + + Move an existing *key* to either end of an ordered dictionary. The item + is moved to the right end if *last* is true (the default) or to the + beginning if *last* is false. Raises :exc:`KeyError` if the *key* does + not exist:: + + >>> d = OrderedDict.fromkeys('abcde') + >>> d.move_to_end('b') + >>> ''.join(d.keys()) + 'acdeb' + >>> d.move_to_end('b', last=False) + >>> ''.join(d.keys()) + 'bacde' + + .. versionadded:: 3.2 + +In addition to the usual mapping methods, ordered dictionaries also support +reverse iteration using :func:`reversed`. + +Equality tests between :class:`OrderedDict` objects are order-sensitive +and are implemented as ``list(od1.items())==list(od2.items())``. +Equality tests between :class:`OrderedDict` objects and other +:class:`~collections.abc.Mapping` objects are order-insensitive like regular +dictionaries. This allows :class:`OrderedDict` objects to be substituted +anywhere a regular dictionary is used. + +.. versionchanged:: 3.5 + The items, keys, and values :term:`views ` + of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. + +.. versionchanged:: 3.6 + With the acceptance of :pep:`468`, order is retained for keyword arguments + passed to the :class:`OrderedDict` constructor and its :meth:`update` + method. + +.. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. + + +:class:`OrderedDict` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is straightforward to create an ordered dictionary variant +that remembers the order the keys were *last* inserted. +If a new entry overwrites an existing entry, the +original insertion position is changed and moved to the end:: + + class LastUpdatedOrderedDict(OrderedDict): + 'Store items in the order the keys were last added' + + def __setitem__(self, key, value): + super().__setitem__(key, value) + self.move_to_end(key) + +An :class:`OrderedDict` would also be useful for implementing +variants of :func:`functools.lru_cache`:: + + class LRU(OrderedDict): + 'Limit size, evicting the least recently looked-up key when full' + + def __init__(self, maxsize=128, /, *args, **kwds): + self.maxsize = maxsize + super().__init__(*args, **kwds) + + def __getitem__(self, key): + value = super().__getitem__(key) + self.move_to_end(key) + return value + + def __setitem__(self, key, value): + if key in self: + self.move_to_end(key) + super().__setitem__(key, value) + if len(self) > self.maxsize: + oldest = next(iter(self)) + del self[oldest] + + +:class:`UserDict` objects +------------------------- + +The class, :class:`UserDict` acts as a wrapper around dictionary objects. +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`dict`; however, this class can be easier +to work with because the underlying dictionary is accessible as an +attribute. + +.. class:: UserDict([initialdata]) + + Class that simulates a dictionary. The instance's contents are kept in a + regular dictionary, which is accessible via the :attr:`data` attribute of + :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is + initialized with its contents; note that a reference to *initialdata* will not + be kept, allowing it be used for other purposes. + + In addition to supporting the methods and operations of mappings, + :class:`UserDict` instances provide the following attribute: + + .. attribute:: data + + A real dictionary used to store the contents of the :class:`UserDict` + class. + + + +:class:`UserList` objects +------------------------- + +This class acts as a wrapper around list objects. It is a useful base class +for your own list-like classes which can inherit from them and override +existing methods or add new ones. In this way, one can add new behaviors to +lists. + +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`list`; however, this class can be easier +to work with because the underlying list is accessible as an attribute. + +.. class:: UserList([list]) + + Class that simulates a list. The instance's contents are kept in a regular + list, which is accessible via the :attr:`data` attribute of :class:`UserList` + instances. The instance's contents are initially set to a copy of *list*, + defaulting to the empty list ``[]``. *list* can be any iterable, for + example a real Python list or a :class:`UserList` object. + + In addition to supporting the methods and operations of mutable sequences, + :class:`UserList` instances provide the following attribute: + + .. attribute:: data + + A real :class:`list` object used to store the contents of the + :class:`UserList` class. + +**Subclassing requirements:** Subclasses of :class:`UserList` are expected to +offer a constructor which can be called with either no arguments or one +argument. List operations which return a new sequence attempt to create an +instance of the actual implementation class. To do so, it assumes that the +constructor can be called with a single parameter, which is a sequence object +used as a data source. + +If a derived class does not wish to comply with this requirement, all of the +special methods supported by this class will need to be overridden; please +consult the sources for information about the methods which need to be provided +in that case. + +:class:`UserString` objects +--------------------------- + +The class, :class:`UserString` acts as a wrapper around string objects. +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`str`; however, this class can be easier +to work with because the underlying string is accessible as an +attribute. + +.. class:: UserString(seq) + + Class that simulates a string object. The instance's + content is kept in a regular string object, which is accessible via the + :attr:`data` attribute of :class:`UserString` instances. The instance's + contents are initially set to a copy of *seq*. The *seq* argument can + be any object which can be converted into a string using the built-in + :func:`str` function. + + In addition to supporting the methods and operations of strings, + :class:`UserString` instances provide the following attribute: + + .. attribute:: data + + A real :class:`str` object used to store the contents of the + :class:`UserString` class. + + .. versionchanged:: 3.5 + New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, + ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/dataclasses.rst.bak b/Doc/library/dataclasses.rst.bak new file mode 100644 index 000000000000000..e706f7fcc566d89 --- /dev/null +++ b/Doc/library/dataclasses.rst.bak @@ -0,0 +1,595 @@ +:mod:`dataclasses` --- Data Classes +=================================== + +.. module:: dataclasses + :synopsis: Generate special methods on user-defined classes. + +.. moduleauthor:: Eric V. Smith +.. sectionauthor:: Eric V. Smith + +**Source code:** :source:`Lib/dataclasses.py` + +-------------- + +This module provides a decorator and functions for automatically +adding generated :term:`special method`\s such as :meth:`__init__` and +:meth:`__repr__` to user-defined classes. It was originally described +in :pep:`557`. + +The member variables to use in these generated methods are defined +using :pep:`526` type annotations. For example this code:: + + from dataclasses import dataclass + + @dataclass + class InventoryItem: + """Class for keeping track of an item in inventory.""" + name: str + unit_price: float + quantity_on_hand: int = 0 + + def total_cost(self) -> float: + return self.unit_price * self.quantity_on_hand + +Will add, among other things, a :meth:`__init__` that looks like:: + + def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): + self.name = name + self.unit_price = unit_price + self.quantity_on_hand = quantity_on_hand + +Note that this method is automatically added to the class: it is not +directly specified in the ``InventoryItem`` definition shown above. + +.. versionadded:: 3.7 + +Module-level decorators, classes, and functions +----------------------------------------------- + +.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + + This function is a :term:`decorator` that is used to add generated + :term:`special method`\s to classes, as described below. + + The :func:`dataclass` decorator examines the class to find + ``field``\s. A ``field`` is defined as class variable that has a + :term:`type annotation `. With two + exceptions described below, nothing in :func:`dataclass` + examines the type specified in the variable annotation. + + The order of the fields in all of the generated methods is the + order in which they appear in the class definition. + + The :func:`dataclass` decorator will add various "dunder" methods to + the class, described below. If any of the added methods already + exist on the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that is called on; no new + class is created. + + If :func:`dataclass` is used just as a simple decorator with no parameters, + it acts as if it has the default values documented in this + signature. That is, these three uses of :func:`dataclass` are + equivalent:: + + @dataclass + class C: + ... + + @dataclass() + class C: + ... + + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + class C: + ... + + The parameters to :func:`dataclass` are: + + - ``init``: If true (the default), a :meth:`__init__` method will be + generated. + + If the class already defines :meth:`__init__`, this parameter is + ignored. + + - ``repr``: If true (the default), a :meth:`__repr__` method will be + generated. The generated repr string will have the class name and + the name and repr of each field, in the order they are defined in + the class. Fields that are marked as being excluded from the repr + are not included. For example: + ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. + + If the class already defines :meth:`__repr__`, this parameter is + ignored. + + - ``eq``: If true (the default), an :meth:`__eq__` method will be + generated. This method compares the class as if it were a tuple + of its fields, in order. Both instances in the comparison must + be of the identical type. + + If the class already defines :meth:`__eq__`, this parameter is + ignored. + + - ``order``: If true (the default is ``False``), :meth:`__lt__`, + :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be + generated. These compare the class as if it were a tuple of its + fields, in order. Both instances in the comparison must be of the + identical type. If ``order`` is true and ``eq`` is false, a + :exc:`ValueError` is raised. + + If the class already defines any of :meth:`__lt__`, + :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then + :exc:`TypeError` is raised. + + - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method + is generated according to how ``eq`` and ``frozen`` are set. + + :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are + added to hashed collections such as dictionaries and sets. Having a + :meth:`__hash__` implies that instances of the class are immutable. + Mutability is a complicated property that depends on the programmer's + intent, the existence and behavior of :meth:`__eq__`, and the values of + the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + + By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` + method unless it is safe to do so. Neither will it add or change an + existing explicitly defined :meth:`__hash__` method. Setting the class + attribute ``__hash__ = None`` has a specific meaning to Python, as + described in the :meth:`__hash__` documentation. + + If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, + then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. + Although not recommended, you can force :func:`dataclass` to create a + :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case + if your class is logically immutable but can nonetheless be mutated. + This is a specialized use case and should be considered carefully. + + Here are the rules governing implicit creation of a :meth:`__hash__` + method. Note that you cannot both have an explicit :meth:`__hash__` + method in your dataclass and set ``unsafe_hash=True``; this will result + in a :exc:`TypeError`. + + If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will + generate a :meth:`__hash__` method for you. If ``eq`` is true and + ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it + unhashable (which it is, since it is mutable). If ``eq`` is false, + :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` + method of the superclass will be used (if the superclass is + :class:`object`, this means it will fall back to id-based hashing). + + - ``frozen``: If true (the default is ``False``), assigning to fields will + generate an exception. This emulates read-only frozen instances. If + :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then + :exc:`TypeError` is raised. See the discussion below. + + ``field``\s may optionally specify a default value, using normal + Python syntax:: + + @dataclass + class C: + a: int # 'a' has no default value + b: int = 0 # assign a default value for 'b' + + In this example, both ``a`` and ``b`` will be included in the added + :meth:`__init__` method, which will be defined as:: + + def __init__(self, a: int, b: int = 0): + + :exc:`TypeError` will be raised if a field without a default value + follows a field with a default value. This is true either when this + occurs in a single class, or as a result of class inheritance. + +.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) + + For common and simple use cases, no other functionality is + required. There are, however, some dataclass features that + require additional per-field information. To satisfy this need for + additional information, you can replace the default field value + with a call to the provided :func:`field` function. For example:: + + @dataclass + class C: + mylist: list[int] = field(default_factory=list) + + c = C() + c.mylist += [1, 2, 3] + + As shown above, the ``MISSING`` value is a sentinel object used to + detect if the ``default`` and ``default_factory`` parameters are + provided. This sentinel is used because ``None`` is a valid value + for ``default``. No code should directly use the ``MISSING`` + value. + + The parameters to :func:`field` are: + + - ``default``: If provided, this will be the default value for this + field. This is needed because the :meth:`field` call itself + replaces the normal position of the default value. + + - ``default_factory``: If provided, it must be a zero-argument + callable that will be called when a default value is needed for + this field. Among other purposes, this can be used to specify + fields with mutable default values, as discussed below. It is an + error to specify both ``default`` and ``default_factory``. + + - ``init``: If true (the default), this field is included as a + parameter to the generated :meth:`__init__` method. + + - ``repr``: If true (the default), this field is included in the + string returned by the generated :meth:`__repr__` method. + + - ``compare``: If true (the default), this field is included in the + generated equality and comparison methods (:meth:`__eq__`, + :meth:`__gt__`, et al.). + + - ``hash``: This can be a bool or ``None``. If true, this field is + included in the generated :meth:`__hash__` method. If ``None`` (the + default), use the value of ``compare``: this would normally be + the expected behavior. A field should be considered in the hash + if it's used for comparisons. Setting this value to anything + other than ``None`` is discouraged. + + One possible reason to set ``hash=False`` but ``compare=True`` + would be if a field is expensive to compute a hash value for, + that field is needed for equality testing, and there are other + fields that contribute to the type's hash value. Even if a field + is excluded from the hash, it will still be used for comparisons. + + - ``metadata``: This can be a mapping or None. None is treated as + an empty dict. This value is wrapped in + :func:`~types.MappingProxyType` to make it read-only, and exposed + on the :class:`Field` object. It is not used at all by Data + Classes, and is provided as a third-party extension mechanism. + Multiple third-parties can each have their own key, to use as a + namespace in the metadata. + + If the default value of a field is specified by a call to + :func:`field()`, then the class attribute for this field will be + replaced by the specified ``default`` value. If no ``default`` is + provided, then the class attribute will be deleted. The intent is + that after the :func:`dataclass` decorator runs, the class + attributes will all contain the default values for the fields, just + as if the default value itself were specified. For example, + after:: + + @dataclass + class C: + x: int + y: int = field(repr=False) + z: int = field(repr=False, default=10) + t: int = 20 + + The class attribute ``C.z`` will be ``10``, the class attribute + ``C.t`` will be ``20``, and the class attributes ``C.x`` and + ``C.y`` will not be set. + +.. class:: Field + + :class:`Field` objects describe each defined field. These objects + are created internally, and are returned by the :func:`fields` + module-level method (see below). Users should never instantiate a + :class:`Field` object directly. Its documented attributes are: + + - ``name``: The name of the field. + + - ``type``: The type of the field. + + - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, + ``compare``, and ``metadata`` have the identical meaning and + values as they do in the :func:`field` declaration. + + Other attributes may exist, but they are private and must not be + inspected or relied on. + +.. function:: fields(class_or_instance) + + Returns a tuple of :class:`Field` objects that define the fields for this + dataclass. Accepts either a dataclass, or an instance of a dataclass. + Raises :exc:`TypeError` if not passed a dataclass or instance of one. + Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. + +.. function:: asdict(instance, *, dict_factory=dict) + + Converts the dataclass ``instance`` to a dict (by using the + factory function ``dict_factory``). Each dataclass is converted + to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, + lists, and tuples are recursed into. For example:: + + @dataclass + class Point: + x: int + y: int + + @dataclass + class C: + mylist: list[Point] + + p = Point(10, 20) + assert asdict(p) == {'x': 10, 'y': 20} + + c = C([Point(0, 0), Point(10, 4)]) + assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} + + Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + +.. function:: astuple(instance, *, tuple_factory=tuple) + + Converts the dataclass ``instance`` to a tuple (by using the + factory function ``tuple_factory``). Each dataclass is converted + to a tuple of its field values. dataclasses, dicts, lists, and + tuples are recursed into. + + Continuing from the previous example:: + + assert astuple(p) == (10, 20) + assert astuple(c) == ([(0, 0), (10, 4)],) + + Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + + Creates a new dataclass with name ``cls_name``, fields as defined + in ``fields``, base classes as given in ``bases``, and initialized + with a namespace as given in ``namespace``. ``fields`` is an + iterable whose elements are each either ``name``, ``(name, type)``, + or ``(name, type, Field)``. If just ``name`` is supplied, + ``typing.Any`` is used for ``type``. The values of ``init``, + ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have + the same meaning as they do in :func:`dataclass`. + + This function is not strictly required, because any Python + mechanism for creating a new class with ``__annotations__`` can + then apply the :func:`dataclass` function to convert that class to + a dataclass. This function is provided as a convenience. For + example:: + + C = make_dataclass('C', + [('x', int), + 'y', + ('z', int, field(default=5))], + namespace={'add_one': lambda self: self.x + 1}) + + Is equivalent to:: + + @dataclass + class C: + x: int + y: 'typing.Any' + z: int = 5 + + def add_one(self): + return self.x + 1 + +.. function:: replace(instance, /, **changes) + + Creates a new object of the same type of ``instance``, replacing + fields with values from ``changes``. If ``instance`` is not a Data + Class, raises :exc:`TypeError`. If values in ``changes`` do not + specify fields, raises :exc:`TypeError`. + + The newly returned object is created by calling the :meth:`__init__` + method of the dataclass. This ensures that + :meth:`__post_init__`, if present, is also called. + + Init-only variables without default values, if any exist, must be + specified on the call to :func:`replace` so that they can be passed to + :meth:`__init__` and :meth:`__post_init__`. + + It is an error for ``changes`` to contain any fields that are + defined as having ``init=False``. A :exc:`ValueError` will be raised + in this case. + + Be forewarned about how ``init=False`` fields work during a call to + :func:`replace`. They are not copied from the source object, but + rather are initialized in :meth:`__post_init__`, if they're + initialized at all. It is expected that ``init=False`` fields will + be rarely and judiciously used. If they are used, it might be wise + to have alternate class constructors, or perhaps a custom + ``replace()`` (or similarly named) method which handles instance + copying. + +.. function:: is_dataclass(class_or_instance) + + Return ``True`` if its parameter is a dataclass or an instance of one, + otherwise return ``False``. + + If you need to know if a class is an instance of a dataclass (and + not a dataclass itself), then add a further check for ``not + isinstance(obj, type)``:: + + def is_dataclass_instance(obj): + return is_dataclass(obj) and not isinstance(obj, type) + +Post-init processing +-------------------- + +The generated :meth:`__init__` code will call a method named +:meth:`__post_init__`, if :meth:`__post_init__` is defined on the +class. It will normally be called as ``self.__post_init__()``. +However, if any ``InitVar`` fields are defined, they will also be +passed to :meth:`__post_init__` in the order they were defined in the +class. If no :meth:`__init__` method is generated, then +:meth:`__post_init__` will not automatically be called. + +Among other uses, this allows for initializing field values that +depend on one or more other fields. For example:: + + @dataclass + class C: + a: float + b: float + c: float = field(init=False) + + def __post_init__(self): + self.c = self.a + self.b + +See the section below on init-only variables for ways to pass +parameters to :meth:`__post_init__`. Also see the warning about how +:func:`replace` handles ``init=False`` fields. + +Class variables +--------------- + +One of two places where :func:`dataclass` actually inspects the type +of a field is to determine if a field is a class variable as defined +in :pep:`526`. It does this by checking if the type of the field is +``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded +from consideration as a field and is ignored by the dataclass +mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the +module-level :func:`fields` function. + +Init-only variables +------------------- + +The other place where :func:`dataclass` inspects a type annotation is to +determine if a field is an init-only variable. It does this by seeing +if the type of a field is of type ``dataclasses.InitVar``. If a field +is an ``InitVar``, it is considered a pseudo-field called an init-only +field. As it is not a true field, it is not returned by the +module-level :func:`fields` function. Init-only fields are added as +parameters to the generated :meth:`__init__` method, and are passed to +the optional :meth:`__post_init__` method. They are not otherwise used +by dataclasses. + +For example, suppose a field will be initialized from a database, if a +value is not provided when creating the class:: + + @dataclass + class C: + i: int + j: int = None + database: InitVar[DatabaseType] = None + + def __post_init__(self, database): + if self.j is None and database is not None: + self.j = database.lookup('j') + + c = C(10, database=my_database) + +In this case, :func:`fields` will return :class:`Field` objects for ``i`` and +``j``, but not for ``database``. + +Frozen instances +---------------- + +It is not possible to create truly immutable Python objects. However, +by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +emulate immutability. In that case, dataclasses will add +:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These +methods will raise a :exc:`FrozenInstanceError` when invoked. + +There is a tiny performance penalty when using ``frozen=True``: +:meth:`__init__` cannot use simple assignment to initialize fields, and +must use :meth:`object.__setattr__`. + +Inheritance +----------- + +When the dataclass is being created by the :meth:`dataclass` decorator, +it looks through all of the class's base classes in reverse MRO (that +is, starting at :class:`object`) and, for each dataclass that it finds, +adds the fields from that base class to an ordered mapping of fields. +After all of the base class fields are added, it adds its own fields +to the ordered mapping. All of the generated methods will use this +combined, calculated ordered mapping of fields. Because the fields +are in insertion order, derived classes override base classes. An +example:: + + @dataclass + class Base: + x: Any = 15.0 + y: int = 0 + + @dataclass + class C(Base): + z: int = 10 + x: int = 15 + +The final list of fields is, in order, ``x``, ``y``, ``z``. The final +type of ``x`` is ``int``, as specified in class ``C``. + +The generated :meth:`__init__` method for ``C`` will look like:: + + def __init__(self, x: int = 15, y: int = 0, z: int = 10): + +Default factory functions +------------------------- + + If a :func:`field` specifies a ``default_factory``, it is called with + zero arguments when a default value for the field is needed. For + example, to create a new instance of a list, use:: + + mylist: list = field(default_factory=list) + + If a field is excluded from :meth:`__init__` (using ``init=False``) + and the field also specifies ``default_factory``, then the default + factory function will always be called from the generated + :meth:`__init__` function. This happens because there is no other + way to give the field an initial value. + +Mutable default values +---------------------- + + Python stores default member variable values in class attributes. + Consider this example, not using dataclasses:: + + class C: + x = [] + def add(self, element): + self.x.append(element) + + o1 = C() + o2 = C() + o1.add(1) + o2.add(2) + assert o1.x == [1, 2] + assert o1.x is o2.x + + Note that the two instances of class ``C`` share the same class + variable ``x``, as expected. + + Using dataclasses, *if* this code was valid:: + + @dataclass + class D: + x: List = [] + def add(self, element): + self.x += element + + it would generate code similar to:: + + class D: + x = [] + def __init__(self, x=x): + self.x = x + def add(self, element): + self.x += element + + assert D().x is D().x + + This has the same issue as the original example using class ``C``. + That is, two instances of class ``D`` that do not specify a value for + ``x`` when creating a class instance will share the same copy of + ``x``. Because dataclasses just use normal Python class creation + they also share this behavior. There is no general way for Data + Classes to detect this condition. Instead, dataclasses will raise a + :exc:`TypeError` if it detects a default parameter of type ``list``, + ``dict``, or ``set``. This is a partial solution, but it does protect + against many common errors. + + Using default factory functions is a way to create new instances of + mutable types as default values for fields:: + + @dataclass + class D: + x: list = field(default_factory=list) + + assert D().x is not D().x + +Exceptions +---------- + +.. exception:: FrozenInstanceError + + Raised when an implicitly defined :meth:`__setattr__` or + :meth:`__delattr__` is called on a dataclass which was defined with + ``frozen=True``. diff --git a/Doc/library/gc.rst.bak b/Doc/library/gc.rst.bak new file mode 100644 index 000000000000000..a3d201d5055c8ec --- /dev/null +++ b/Doc/library/gc.rst.bak @@ -0,0 +1,322 @@ +:mod:`gc` --- Garbage Collector interface +========================================= + +.. module:: gc + :synopsis: Interface to the cycle-detecting garbage collector. + +.. moduleauthor:: Neil Schemenauer +.. sectionauthor:: Neil Schemenauer + +-------------- + +This module provides an interface to the optional garbage collector. It +provides the ability to disable the collector, tune the collection frequency, +and set debugging options. It also provides access to unreachable objects that +the collector found but cannot free. Since the collector supplements the +reference counting already used in Python, you can disable the collector if you +are sure your program does not create reference cycles. Automatic collection +can be disabled by calling ``gc.disable()``. To debug a leaking program call +``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes +``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in +gc.garbage for inspection. + +The :mod:`gc` module provides the following functions: + + +.. function:: enable() + + Enable automatic garbage collection. + + +.. function:: disable() + + Disable automatic garbage collection. + + +.. function:: isenabled() + + Return ``True`` if automatic collection is enabled. + + +.. function:: collect(generation=2) + + With no arguments, run a full collection. The optional argument *generation* + may be an integer specifying which generation to collect (from 0 to 2). A + :exc:`ValueError` is raised if the generation number is invalid. The number of + unreachable objects found is returned. + + The free lists maintained for a number of built-in types are cleared + whenever a full collection or collection of the highest generation (2) + is run. Not all items in some free lists may be freed due to the + particular implementation, in particular :class:`float`. + + +.. function:: set_debug(flags) + + Set the garbage collection debugging flags. Debugging information will be + written to ``sys.stderr``. See below for a list of debugging flags which can be + combined using bit operations to control debugging. + + +.. function:: get_debug() + + Return the debugging flags currently set. + + +.. function:: get_objects(generation=None) + + Returns a list of all objects tracked by the collector, excluding the list + returned. If *generation* is not None, return only the objects tracked by + the collector that are in that generation. + + .. versionchanged:: 3.8 + New *generation* parameter. + +.. function:: get_stats() + + Return a list of three per-generation dictionaries containing collection + statistics since interpreter start. The number of keys may change + in the future, but currently each dictionary will contain the following + items: + + * ``collections`` is the number of times this generation was collected; + + * ``collected`` is the total number of objects collected inside this + generation; + + * ``uncollectable`` is the total number of objects which were found + to be uncollectable (and were therefore moved to the :data:`garbage` + list) inside this generation. + + .. versionadded:: 3.4 + + +.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) + + Set the garbage collection thresholds (the collection frequency). Setting + *threshold0* to zero disables collection. + + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object + allocations and deallocations since the last collection. When the number of + allocations minus the number of deallocations exceeds *threshold0*, collection + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. + + +.. function:: get_count() + + Return the current collection counts as a tuple of ``(count0, count1, + count2)``. + + +.. function:: get_threshold() + + Return the current collection thresholds as a tuple of ``(threshold0, + threshold1, threshold2)``. + + +.. function:: get_referrers(*objs) + + Return the list of objects that directly refer to any of objs. This function + will only locate those containers which support garbage collection; extension + types which do refer to other objects but do not support garbage collection will + not be found. + + Note that objects which have already been dereferenced, but which live in cycles + and have not yet been collected by the garbage collector can be listed among the + resulting referrers. To get only currently live objects, call :func:`collect` + before calling :func:`get_referrers`. + + .. warning:: + Care must be taken when using objects returned by :func:`get_referrers` because + some of them could still be under construction and hence in a temporarily + invalid state. Avoid using :func:`get_referrers` for any purpose other than + debugging. + + +.. function:: get_referents(*objs) + + Return a list of objects directly referred to by any of the arguments. The + referents returned are those objects visited by the arguments' C-level + :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually + directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects + that support garbage collection, and are only required to visit objects that may + be involved in a cycle. So, for example, if an integer is directly reachable + from an argument, that integer object may or may not appear in the result list. + + +.. function:: is_tracked(obj) + + Returns ``True`` if the object is currently tracked by the garbage collector, + ``False`` otherwise. As a general rule, instances of atomic types aren't + tracked and instances of non-atomic types (containers, user-defined + objects...) are. However, some type-specific optimizations can be present + in order to suppress the garbage collector footprint of simple instances + (e.g. dicts containing only atomic keys and values):: + + >>> gc.is_tracked(0) + False + >>> gc.is_tracked("a") + False + >>> gc.is_tracked([]) + True + >>> gc.is_tracked({}) + False + >>> gc.is_tracked({"a": 1}) + False + >>> gc.is_tracked({"a": []}) + True + + .. versionadded:: 3.1 + + +.. function:: is_finalized(obj) + + Returns ``True`` if the given object has been finalized by the + garbage collector, ``False`` otherwise. :: + + >>> x = None + >>> class Lazarus: + ... def __del__(self): + ... global x + ... x = self + ... + >>> lazarus = Lazarus() + >>> gc.is_finalized(lazarus) + False + >>> del lazarus + >>> gc.is_finalized(x) + True + + .. versionadded:: 3.9 + + +.. function:: freeze() + + Freeze all the objects tracked by gc - move them to a permanent generation + and ignore all the future collections. This can be used before a POSIX + fork() call to make the gc copy-on-write friendly or to speed up collection. + Also collection before a POSIX fork() call may free pages for future + allocation which can cause copy-on-write too so it's advised to disable gc + in parent process and freeze before fork and enable gc in child process. + + .. versionadded:: 3.7 + + +.. function:: unfreeze() + + Unfreeze the objects in the permanent generation, put them back into the + oldest generation. + + .. versionadded:: 3.7 + + +.. function:: get_freeze_count() + + Return the number of objects in the permanent generation. + + .. versionadded:: 3.7 + + +The following variables are provided for read-only access (you can mutate the +values but should not rebind them): + +.. data:: garbage + + A list of objects which the collector found to be unreachable but could + not be freed (uncollectable objects). Starting with Python 3.4, this + list should be empty most of the time, except when using instances of + C extension types with a non-``NULL`` ``tp_del`` slot. + + If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be + added to this list rather than freed. + + .. versionchanged:: 3.2 + If this list is non-empty at :term:`interpreter shutdown`, a + :exc:`ResourceWarning` is emitted, which is silent by default. If + :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects + are printed. + + .. versionchanged:: 3.4 + Following :pep:`442`, objects with a :meth:`__del__` method don't end + up in :attr:`gc.garbage` anymore. + +.. data:: callbacks + + A list of callbacks that will be invoked by the garbage collector before and + after collection. The callbacks will be called with two arguments, + *phase* and *info*. + + *phase* can be one of two values: + + "start": The garbage collection is about to start. + + "stop": The garbage collection has finished. + + *info* is a dict providing more information for the callback. The following + keys are currently defined: + + "generation": The oldest generation being collected. + + "collected": When *phase* is "stop", the number of objects + successfully collected. + + "uncollectable": When *phase* is "stop", the number of objects + that could not be collected and were put in :data:`garbage`. + + Applications can add their own callbacks to this list. The primary + use cases are: + + Gathering statistics about garbage collection, such as how often + various generations are collected, and how long the collection + takes. + + Allowing applications to identify and clear their own uncollectable + types when they appear in :data:`garbage`. + + .. versionadded:: 3.3 + + +The following constants are provided for use with :func:`set_debug`: + + +.. data:: DEBUG_STATS + + Print statistics during collection. This information can be useful when tuning + the collection frequency. + + +.. data:: DEBUG_COLLECTABLE + + Print information on collectable objects found. + + +.. data:: DEBUG_UNCOLLECTABLE + + Print information of uncollectable objects found (objects which are not + reachable but cannot be freed by the collector). These objects will be added + to the ``garbage`` list. + + .. versionchanged:: 3.2 + Also print the contents of the :data:`garbage` list at + :term:`interpreter shutdown`, if it isn't empty. + +.. data:: DEBUG_SAVEALL + + When set, all unreachable objects found will be appended to *garbage* rather + than being freed. This can be useful for debugging a leaking program. + + +.. data:: DEBUG_LEAK + + The debugging flags necessary for the collector to print information about a + leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | + DEBUG_SAVEALL``). diff --git a/Doc/library/importlib.metadata.rst.bak b/Doc/library/importlib.metadata.rst.bak new file mode 100644 index 000000000000000..7f154ea02cc4f2d --- /dev/null +++ b/Doc/library/importlib.metadata.rst.bak @@ -0,0 +1,262 @@ +.. _using: + +================================= + Using :mod:`!importlib.metadata` +================================= + +.. note:: + This functionality is provisional and may deviate from the usual + version semantics of the standard library. + +``importlib.metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +:mod:`importlib.resources` in Python 3.7 +and newer (backported as `importlib_resources`_ for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by :pep:`566` or its older specifications. +By default, package metadata can live on the file system or in zip archives on +:data:`sys.path`. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it: + +.. code-block:: shell-session + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following: + +.. code-block:: pycon + + (example) $ python + >>> from importlib.metadata import version # doctest: +SKIP + >>> version('wheel') # doctest: +SKIP + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint ` objects. + +You can get the :ref:`metadata for a distribution `:: + + >>> list(metadata('wheel')) # doctest: +SKIP + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number `, list its +:ref:`constituent files `, and get a list of the distribution's +:ref:`requirements`. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: + + >>> eps = entry_points() # doctest: +SKIP + >>> list(eps) # doctest: +SKIP + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] # doctest: +SKIP + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP + >>> wheel # doctest: +SKIP + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module # doctest: +SKIP + 'wheel.cli' + >>> wheel.attr # doctest: +SKIP + 'main' + >>> wheel.extras # doctest: +SKIP + [] + >>> main = wheel.load() # doctest: +SKIP + >>> main # doctest: +SKIP + + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +`_ +for more information on entry points, their definition, and usage. + + +.. _metadata: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') # doctest: +SKIP + +The keys of the returned data structure, a ``PackageMetadata``, +name the metadata keywords, and +the values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') # doctest: +SKIP + '0.32.3' + + +.. _files: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP + >>> util # doctest: +SKIP + PackagePath('wheel/util.py') + >>> util.size # doctest: +SKIP + 859 + >>> util.dist # doctest: +SKIP + + >>> util.hash # doctest: +SKIP + + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) # doctest: +SKIP + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + +In the case where the metadata file listing files +(RECORD or SOURCES.txt) is missing, ``files()`` will +return ``None``. The caller may wish to wrap calls to +``files()`` in `always_iterable +`_ +or otherwise guard against this condition if the target +distribution is not known to have the metadata present. + +.. _requirements: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function:: + + >>> requires('wheel') # doctest: +SKIP + ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] + + +Distributions +============= + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> dist.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> dist.metadata['License'] # doctest: +SKIP + 'MIT' + +The full set of available metadata is not described here. See :pep:`566` +for additional details. + + +Extending the search algorithm +============================== + +Because package metadata is not available through :data:`sys.path` searches, or +package loaders directly, the metadata for a package is found through import +system :ref:`finders `. To find a distribution package's metadata, +``importlib.metadata`` queries the list of :term:`meta path finders ` on +:data:`sys.meta_path`. + +The default ``PathFinder`` for Python includes a hook that calls into +``importlib.metadata.MetadataPathFinder`` for finding distributions +loaded from typical file-system-based paths. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib.metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +:data:`sys.meta_path` and presents this extended interface as the +``DistributionFinder`` abstract base class, which defines this abstract +method:: + + @abc.abstractmethod + def find_distributions(context=DistributionFinder.Context()): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages for the indicated ``context``. + """ + +The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` +properties indicating the path to search and name to match and may +supply other relevant context. + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, subclass +``Distribution`` and implement the abstract methods. Then from +a custom finder, return instances of this derived ``Distribution`` in the +``find_distributions()`` method. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html + + +.. rubric:: Footnotes diff --git a/Doc/library/logging.rst.bak b/Doc/library/logging.rst.bak new file mode 100644 index 000000000000000..431a5849fa9bffc --- /dev/null +++ b/Doc/library/logging.rst.bak @@ -0,0 +1,1350 @@ +:mod:`logging` --- Logging facility for Python +============================================== + +.. module:: logging + :synopsis: Flexible event logging system for applications. + +.. moduleauthor:: Vinay Sajip +.. sectionauthor:: Vinay Sajip + +**Source code:** :source:`Lib/logging/__init__.py` + +.. index:: pair: Errors; logging + +.. sidebar:: Important + + This page contains the API reference information. For tutorial + information and discussion of more advanced topics, see + + * :ref:`Basic Tutorial ` + * :ref:`Advanced Tutorial ` + * :ref:`Logging Cookbook ` + +-------------- + +This module defines functions and classes which implement a flexible event +logging system for applications and libraries. + +The key benefit of having the logging API provided by a standard library module +is that all Python modules can participate in logging, so your application log +can include your own messages integrated with messages from third-party +modules. + +The module provides a lot of functionality and flexibility. If you are +unfamiliar with logging, the best way to get to grips with it is to see the +tutorials (see the links on the right). + +The basic classes defined by the module, together with their functions, are +listed below. + +* Loggers expose the interface that application code directly uses. +* Handlers send the log records (created by loggers) to the appropriate + destination. +* Filters provide a finer grained facility for determining which log records + to output. +* Formatters specify the layout of log records in the final output. + + +.. _logger: + +Logger Objects +-------------- + +Loggers have the following attributes and methods. Note that Loggers should +*NEVER* be instantiated directly, but always through the module-level function +``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same +name will always return a reference to the same Logger object. + +The ``name`` is potentially a period-separated hierarchical value, like +``foo.bar.baz`` (though it could also be just plain ``foo``, for example). +Loggers that are further down in the hierarchical list are children of loggers +higher up in the list. For example, given a logger with a name of ``foo``, +loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all +descendants of ``foo``. The logger name hierarchy is analogous to the Python +package hierarchy, and identical to it if you organise your loggers on a +per-module basis using the recommended construction +``logging.getLogger(__name__)``. That's because in a module, ``__name__`` +is the module's name in the Python package namespace. + + +.. class:: Logger + + .. attribute:: Logger.propagate + + If this attribute evaluates to true, events logged to this logger will be + passed to the handlers of higher level (ancestor) loggers, in addition to + any handlers attached to this logger. Messages are passed directly to the + ancestor loggers' handlers - neither the level nor filters of the ancestor + loggers in question are considered. + + If this evaluates to false, logging messages are not passed to the handlers + of ancestor loggers. + + The constructor sets this attribute to ``True``. + + .. note:: If you attach a handler to a logger *and* one or more of its + ancestors, it may emit the same record multiple times. In general, you + should not need to attach a handler to more than one logger - if you just + attach it to the appropriate logger which is highest in the logger + hierarchy, then it will see all events logged by all descendant loggers, + provided that their propagate setting is left set to ``True``. A common + scenario is to attach handlers only to the root logger, and to let + propagation take care of the rest. + + .. method:: Logger.setLevel(level) + + Sets the threshold for this logger to *level*. Logging messages which are less + severe than *level* will be ignored; logging messages which have severity *level* + or higher will be emitted by whichever handler or handlers service this logger, + unless a handler's level has been set to a higher severity level than *level*. + + When a logger is created, the level is set to :const:`NOTSET` (which causes + all messages to be processed when the logger is the root logger, or delegation + to the parent when the logger is a non-root logger). Note that the root logger + is created with level :const:`WARNING`. + + The term 'delegation to the parent' means that if a logger has a level of + NOTSET, its chain of ancestor loggers is traversed until either an ancestor with + a level other than NOTSET is found, or the root is reached. + + If an ancestor is found with a level other than NOTSET, then that ancestor's + level is treated as the effective level of the logger where the ancestor search + began, and is used to determine how a logging event is handled. + + If the root is reached, and it has a level of NOTSET, then all messages will be + processed. Otherwise, the root's level will be used as the effective level. + + See :ref:`levels` for a list of levels. + + .. versionchanged:: 3.2 + The *level* parameter now accepts a string representation of the + level such as 'INFO' as an alternative to the integer constants + such as :const:`INFO`. Note, however, that levels are internally stored + as integers, and methods such as e.g. :meth:`getEffectiveLevel` and + :meth:`isEnabledFor` will return/expect to be passed integers. + + + .. method:: Logger.isEnabledFor(level) + + Indicates if a message of severity *level* would be processed by this logger. + This method checks first the module-level level set by + ``logging.disable(level)`` and then the logger's effective level as determined + by :meth:`getEffectiveLevel`. + + + .. method:: Logger.getEffectiveLevel() + + Indicates the effective level for this logger. If a value other than + :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, + the hierarchy is traversed towards the root until a value other than + :const:`NOTSET` is found, and that value is returned. The value returned is + an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` + etc. + + + .. method:: Logger.getChild(suffix) + + Returns a logger which is a descendant to this logger, as determined by the suffix. + Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same + logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a + convenience method, useful when the parent logger is named using e.g. ``__name__`` + rather than a literal string. + + .. versionadded:: 3.2 + + + .. method:: Logger.debug(msg, *args, **kwargs) + + Logs a message with level :const:`DEBUG` on this logger. The *msg* is the + message format string, and the *args* are the arguments which are merged into + *msg* using the string formatting operator. (Note that this means that you can + use keywords in the format string, together with a single dictionary argument.) + No % formatting operation is performed on *msg* when no *args* are supplied. + + There are four keyword arguments in *kwargs* which are inspected: + *exc_info*, *stack_info*, *stacklevel* and *extra*. + + If *exc_info* does not evaluate as false, it causes exception information to be + added to the logging message. If an exception tuple (in the format returned by + :func:`sys.exc_info`) or an exception instance is provided, it is used; + otherwise, :func:`sys.exc_info` is called to get the exception information. + + The second optional keyword argument is *stack_info*, which defaults to + ``False``. If true, stack information is added to the logging + message, including the actual logging call. Note that this is not the same + stack information as that displayed through specifying *exc_info*: The + former is stack frames from the bottom of the stack up to the logging call + in the current thread, whereas the latter is information about stack frames + which have been unwound, following an exception, while searching for + exception handlers. + + You can specify *stack_info* independently of *exc_info*, e.g. to just show + how you got to a certain point in your code, even when no exceptions were + raised. The stack frames are printed following a header line which says: + + .. code-block:: none + + Stack (most recent call last): + + This mimics the ``Traceback (most recent call last):`` which is used when + displaying exception frames. + + The third optional keyword argument is *stacklevel*, which defaults to ``1``. + If greater than 1, the corresponding number of stack frames are skipped + when computing the line number and function name set in the :class:`LogRecord` + created for the logging event. This can be used in logging helpers so that + the function name, filename and line number recorded are not the information + for the helper function/method, but rather its caller. The name of this + parameter mirrors the equivalent one in the :mod:`warnings` module. + + The fourth keyword argument is *extra* which can be used to pass a + dictionary which is used to populate the __dict__ of the :class:`LogRecord` + created for the logging event with user-defined attributes. These custom + attributes can then be used as you like. For example, they could be + incorporated into logged messages. For example:: + + FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + logging.basicConfig(format=FORMAT) + d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} + logger = logging.getLogger('tcpserver') + logger.warning('Protocol problem: %s', 'connection reset', extra=d) + + would print something like + + .. code-block:: none + + 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset + + The keys in the dictionary passed in *extra* should not clash with the keys used + by the logging system. (See the :class:`Formatter` documentation for more + information on which keys are used by the logging system.) + + If you choose to use these attributes in logged messages, you need to exercise + some care. In the above example, for instance, the :class:`Formatter` has been + set up with a format string which expects 'clientip' and 'user' in the attribute + dictionary of the :class:`LogRecord`. If these are missing, the message will + not be logged because a string formatting exception will occur. So in this case, + you always need to pass the *extra* dictionary with these keys. + + While this might be annoying, this feature is intended for use in specialized + circumstances, such as multi-threaded servers where the same code executes in + many contexts, and interesting conditions which arise are dependent on this + context (such as remote client IP address and authenticated user name, in the + above example). In such circumstances, it is likely that specialized + :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + + .. versionchanged:: 3.2 + The *stack_info* parameter was added. + + .. versionchanged:: 3.5 + The *exc_info* parameter can now accept exception instances. + + .. versionchanged:: 3.8 + The *stacklevel* parameter was added. + + + .. method:: Logger.info(msg, *args, **kwargs) + + Logs a message with level :const:`INFO` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.warning(msg, *args, **kwargs) + + Logs a message with level :const:`WARNING` on this logger. The arguments are + interpreted as for :meth:`debug`. + + .. note:: There is an obsolete method ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + + .. method:: Logger.error(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.critical(msg, *args, **kwargs) + + Logs a message with level :const:`CRITICAL` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.log(level, msg, *args, **kwargs) + + Logs a message with integer level *level* on this logger. The other arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.exception(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on this logger. The arguments are + interpreted as for :meth:`debug`. Exception info is added to the logging + message. This method should only be called from an exception handler. + + + .. method:: Logger.addFilter(filter) + + Adds the specified filter *filter* to this logger. + + + .. method:: Logger.removeFilter(filter) + + Removes the specified filter *filter* from this logger. + + + .. method:: Logger.filter(record) + + Apply this logger's filters to the record and return ``True`` if the + record is to be processed. The filters are consulted in turn, until one of + them returns a false value. If none of them return a false value, the record + will be processed (passed to handlers). If one returns a false value, no + further processing of the record occurs. + + + .. method:: Logger.addHandler(hdlr) + + Adds the specified handler *hdlr* to this logger. + + + .. method:: Logger.removeHandler(hdlr) + + Removes the specified handler *hdlr* from this logger. + + + .. method:: Logger.findCaller(stack_info=False, stacklevel=1) + + Finds the caller's source filename and line number. Returns the filename, line + number, function name and stack information as a 4-element tuple. The stack + information is returned as ``None`` unless *stack_info* is ``True``. + + The *stacklevel* parameter is passed from code calling the :meth:`debug` + and other APIs. If greater than 1, the excess is used to skip stack frames + before determining the values to be returned. This will generally be useful + when calling logging APIs from helper/wrapper code, so that the information + in the event log refers not to the helper/wrapper code, but to the code that + calls it. + + + .. method:: Logger.handle(record) + + Handles a record by passing it to all handlers associated with this logger and + its ancestors (until a false value of *propagate* is found). This method is used + for unpickled records received from a socket, as well as those created locally. + Logger-level filtering is applied using :meth:`~Logger.filter`. + + + .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) + + This is a factory method which can be overridden in subclasses to create + specialized :class:`LogRecord` instances. + + .. method:: Logger.hasHandlers() + + Checks to see if this logger has any handlers configured. This is done by + looking for handlers in this logger and its parents in the logger hierarchy. + Returns ``True`` if a handler was found, else ``False``. The method stops searching + up the hierarchy whenever a logger with the 'propagate' attribute set to + false is found - that will be the last logger which is checked for the + existence of handlers. + + .. versionadded:: 3.2 + + .. versionchanged:: 3.7 + Loggers can now be pickled and unpickled. + +.. _levels: + +Logging Levels +-------------- + +The numeric values of logging levels are given in the following table. These are +primarily of interest if you want to define your own levels, and need them to +have specific values relative to the predefined levels. If you define a level +with the same numeric value, it overwrites the predefined value; the predefined +name is lost. + ++--------------+---------------+ +| Level | Numeric value | ++==============+===============+ +| ``CRITICAL`` | 50 | ++--------------+---------------+ +| ``ERROR`` | 40 | ++--------------+---------------+ +| ``WARNING`` | 30 | ++--------------+---------------+ +| ``INFO`` | 20 | ++--------------+---------------+ +| ``DEBUG`` | 10 | ++--------------+---------------+ +| ``NOTSET`` | 0 | ++--------------+---------------+ + + +.. _handler: + +Handler Objects +--------------- + +Handlers have the following attributes and methods. Note that :class:`Handler` +is never instantiated directly; this class acts as a base for more useful +subclasses. However, the :meth:`__init__` method in subclasses needs to call +:meth:`Handler.__init__`. + +.. class:: Handler + + .. method:: Handler.__init__(level=NOTSET) + + Initializes the :class:`Handler` instance by setting its level, setting the list + of filters to the empty list and creating a lock (using :meth:`createLock`) for + serializing access to an I/O mechanism. + + + .. method:: Handler.createLock() + + Initializes a thread lock which can be used to serialize access to underlying + I/O functionality which may not be threadsafe. + + + .. method:: Handler.acquire() + + Acquires the thread lock created with :meth:`createLock`. + + + .. method:: Handler.release() + + Releases the thread lock acquired with :meth:`acquire`. + + + .. method:: Handler.setLevel(level) + + Sets the threshold for this handler to *level*. Logging messages which are + less severe than *level* will be ignored. When a handler is created, the + level is set to :const:`NOTSET` (which causes all messages to be + processed). + + See :ref:`levels` for a list of levels. + + .. versionchanged:: 3.2 + The *level* parameter now accepts a string representation of the + level such as 'INFO' as an alternative to the integer constants + such as :const:`INFO`. + + + .. method:: Handler.setFormatter(fmt) + + Sets the :class:`Formatter` for this handler to *fmt*. + + + .. method:: Handler.addFilter(filter) + + Adds the specified filter *filter* to this handler. + + + .. method:: Handler.removeFilter(filter) + + Removes the specified filter *filter* from this handler. + + + .. method:: Handler.filter(record) + + Apply this handler's filters to the record and return ``True`` if the + record is to be processed. The filters are consulted in turn, until one of + them returns a false value. If none of them return a false value, the record + will be emitted. If one returns a false value, the handler will not emit the + record. + + + .. method:: Handler.flush() + + Ensure all logging output has been flushed. This version does nothing and is + intended to be implemented by subclasses. + + + .. method:: Handler.close() + + Tidy up any resources used by the handler. This version does no output but + removes the handler from an internal list of handlers which is closed when + :func:`shutdown` is called. Subclasses should ensure that this gets called + from overridden :meth:`close` methods. + + + .. method:: Handler.handle(record) + + Conditionally emits the specified logging record, depending on filters which may + have been added to the handler. Wraps the actual emission of the record with + acquisition/release of the I/O thread lock. + + + .. method:: Handler.handleError(record) + + This method should be called from handlers when an exception is encountered + during an :meth:`emit` call. If the module-level attribute + ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + what is mostly wanted for a logging system - most users will not care about + errors in the logging system, they are more interested in application + errors. You could, however, replace this with a custom handler if you wish. + The specified record is the one which was being processed when the exception + occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + more useful during development). + + + .. method:: Handler.format(record) + + Do formatting for a record - if a formatter is set, use it. Otherwise, use the + default formatter for the module. + + + .. method:: Handler.emit(record) + + Do whatever it takes to actually log the specified logging record. This version + is intended to be implemented by subclasses and so raises a + :exc:`NotImplementedError`. + +For a list of handlers included as standard, see :mod:`logging.handlers`. + +.. _formatter-objects: + +Formatter Objects +----------------- + +.. currentmodule:: logging + +:class:`Formatter` objects have the following attributes and methods. They are +responsible for converting a :class:`LogRecord` to (usually) a string which can +be interpreted by either a human or an external system. The base +:class:`Formatter` allows a formatting string to be specified. If none is +supplied, the default value of ``'%(message)s'`` is used, which just includes +the message in the logging call. To have additional items of information in the +formatted output (such as a timestamp), keep reading. + +A Formatter can be initialized with a format string which makes use of knowledge +of the :class:`LogRecord` attributes - such as the default value mentioned above +making use of the fact that the user's message and arguments are pre-formatted +into a :class:`LogRecord`'s *message* attribute. This format string contains +standard Python %-style mapping keys. See section :ref:`old-string-formatting` +for more information on string formatting. + +The useful mapping keys in a :class:`LogRecord` are given in the section on +:ref:`logrecord-attributes`. + + +.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) + + Returns a new instance of the :class:`Formatter` class. The instance is + initialized with a format string for the message as a whole, as well as a + format string for the date/time portion of a message. If no *fmt* is + specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format + is used which is described in the :meth:`formatTime` documentation. + + The *style* parameter can be one of '%', '{' or '$' and determines how + the format string will be merged with its data: using one of %-formatting, + :meth:`str.format` or :class:`string.Template`. This only applies to the + format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the + actual log messages passed to ``Logger.debug`` etc; see + :ref:`formatting-styles` for more information on using {- and $-formatting + for log messages. + + The *defaults* parameter can be a dictionary with default values to use in + custom fields. For example: + ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` + + .. versionchanged:: 3.2 + The *style* parameter was added. + + .. versionchanged:: 3.8 + The *validate* parameter was added. Incorrect or mismatched style and fmt + will raise a ``ValueError``. + For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. + + .. versionchanged:: 3.10 + The *defaults* parameter was added. + + .. method:: format(record) + + The record's attribute dictionary is used as the operand to a string + formatting operation. Returns the resulting string. Before formatting the + dictionary, a couple of preparatory steps are carried out. The *message* + attribute of the record is computed using *msg* % *args*. If the + formatting string contains ``'(asctime)'``, :meth:`formatTime` is called + to format the event time. If there is exception information, it is + formatted using :meth:`formatException` and appended to the message. Note + that the formatted exception information is cached in attribute + *exc_text*. This is useful because the exception information can be + pickled and sent across the wire, but you should be careful if you have + more than one :class:`Formatter` subclass which customizes the formatting + of exception information. In this case, you will have to clear the cached + value (by setting the *exc_text* attribute to ``None``) after a formatter + has done its formatting, so that the next formatter to handle the event + doesn't use the cached value, but recalculates it afresh. + + If stack information is available, it's appended after the exception + information, using :meth:`formatStack` to transform it if necessary. + + + .. method:: formatTime(record, datefmt=None) + + This method should be called from :meth:`format` by a formatter which + wants to make use of a formatted time. This method can be overridden in + formatters to provide for any specific requirement, but the basic behavior + is as follows: if *datefmt* (a string) is specified, it is used with + :func:`time.strftime` to format the creation time of the + record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the + uuu part is a millisecond value and the other letters are as per the + :func:`time.strftime` documentation. An example time in this format is + ``2003-01-23 00:29:50,411``. The resulting string is returned. + + This function uses a user-configurable function to convert the creation + time to a tuple. By default, :func:`time.localtime` is used; to change + this for a particular formatter instance, set the ``converter`` attribute + to a function with the same signature as :func:`time.localtime` or + :func:`time.gmtime`. To change it for all formatters, for example if you + want all logging times to be shown in GMT, set the ``converter`` + attribute in the ``Formatter`` class. + + .. versionchanged:: 3.3 + Previously, the default format was hard-coded as in this example: + ``2010-09-06 22:38:15,292`` where the part before the comma is + handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the + part after the comma is a millisecond value. Because strptime does not + have a format placeholder for milliseconds, the millisecond value is + appended using another format string, ``'%s,%03d'`` --- and both of these + format strings have been hardcoded into this method. With the change, + these strings are defined as class-level attributes which can be + overridden at the instance level when desired. The names of the + attributes are ``default_time_format`` (for the strptime format string) + and ``default_msec_format`` (for appending the millisecond value). + + .. versionchanged:: 3.9 + The ``default_msec_format`` can be ``None``. + + .. method:: formatException(exc_info) + + Formats the specified exception information (a standard exception tuple as + returned by :func:`sys.exc_info`) as a string. This default implementation + just uses :func:`traceback.print_exception`. The resulting string is + returned. + + .. method:: formatStack(stack_info) + + Formats the specified stack information (a string as returned by + :func:`traceback.print_stack`, but with the last newline removed) as a + string. This default implementation just returns the input value. + +.. _filter: + +Filter Objects +-------------- + +``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated +filtering than is provided by levels. The base filter class only allows events +which are below a certain point in the logger hierarchy. For example, a filter +initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', +'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the +empty string, all events are passed. + + +.. class:: Filter(name='') + + Returns an instance of the :class:`Filter` class. If *name* is specified, it + names a logger which, together with its children, will have its events allowed + through the filter. If *name* is the empty string, allows every event. + + + .. method:: filter(record) + + Is the specified record to be logged? Returns zero for no, nonzero for + yes. If deemed appropriate, the record may be modified in-place by this + method. + +Note that filters attached to handlers are consulted before an event is +emitted by the handler, whereas filters attached to loggers are consulted +whenever an event is logged (using :meth:`debug`, :meth:`info`, +etc.), before sending an event to handlers. This means that events which have +been generated by descendant loggers will not be filtered by a logger's filter +setting, unless the filter has also been applied to those descendant loggers. + +You don't actually need to subclass ``Filter``: you can pass any instance +which has a ``filter`` method with the same semantics. + +.. versionchanged:: 3.2 + You don't need to create specialized ``Filter`` classes, or use other + classes with a ``filter`` method: you can use a function (or other + callable) as a filter. The filtering logic will check to see if the filter + object has a ``filter`` attribute: if it does, it's assumed to be a + ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's + assumed to be a callable and called with the record as the single + parameter. The returned value should conform to that returned by + :meth:`~Filter.filter`. + +Although filters are used primarily to filter records based on more +sophisticated criteria than levels, they get to see every record which is +processed by the handler or logger they're attached to: this can be useful if +you want to do things like counting how many records were processed by a +particular logger or handler, or adding, changing or removing attributes in +the :class:`LogRecord` being processed. Obviously changing the LogRecord needs +to be done with some care, but it does allow the injection of contextual +information into logs (see :ref:`filters-contextual`). + +.. _log-record: + +LogRecord Objects +----------------- + +:class:`LogRecord` instances are created automatically by the :class:`Logger` +every time something is logged, and can be created manually via +:func:`makeLogRecord` (for example, from a pickled event received over the +wire). + + +.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) + + Contains all the information pertinent to the event being logged. + + The primary information is passed in :attr:`msg` and :attr:`args`, which + are combined using ``msg % args`` to create the :attr:`message` field of the + record. + + :param name: The name of the logger used to log the event represented by + this LogRecord. Note that this name will always have this + value, even though it may be emitted by a handler attached to + a different (ancestor) logger. + :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) + Note that this is converted to *two* attributes of the LogRecord: + ``levelno`` for the numeric value and ``levelname`` for the + corresponding level name. + :param pathname: The full pathname of the source file where the logging call + was made. + :param lineno: The line number in the source file where the logging call was + made. + :param msg: The event description message, possibly a format string with + placeholders for variable data. + :param args: Variable data to merge into the *msg* argument to obtain the + event description. + :param exc_info: An exception tuple with the current exception information, + or ``None`` if no exception information is available. + :param func: The name of the function or method from which the logging call + was invoked. + :param sinfo: A text string representing stack information from the base of + the stack in the current thread, up to the logging call. + + .. method:: getMessage() + + Returns the message for this :class:`LogRecord` instance after merging any + user-supplied arguments with the message. If the user-supplied message + argument to the logging call is not a string, :func:`str` is called on it to + convert it to a string. This allows use of user-defined classes as + messages, whose ``__str__`` method can return the actual format string to + be used. + + .. versionchanged:: 3.2 + The creation of a :class:`LogRecord` has been made more configurable by + providing a factory which is used to create the record. The factory can be + set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` + (see this for the factory's signature). + + This functionality can be used to inject your own values into a + :class:`LogRecord` at creation time. You can use the following pattern:: + + old_factory = logging.getLogRecordFactory() + + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.custom_attribute = 0xdecafbad + return record + + logging.setLogRecordFactory(record_factory) + + With this pattern, multiple factories could be chained, and as long + as they don't overwrite each other's attributes or unintentionally + overwrite the standard attributes listed above, there should be no + surprises. + + +.. _logrecord-attributes: + +LogRecord attributes +-------------------- + +The LogRecord has a number of attributes, most of which are derived from the +parameters to the constructor. (Note that the names do not always correspond +exactly between the LogRecord constructor parameters and the LogRecord +attributes.) These attributes can be used to merge data from the record into +the format string. The following table lists (in alphabetical order) the +attribute names, their meanings and the corresponding placeholder in a %-style +format string. + +If you are using {}-formatting (:func:`str.format`), you can use +``{attrname}`` as the placeholder in the format string. If you are using +$-formatting (:class:`string.Template`), use the form ``${attrname}``. In +both cases, of course, replace ``attrname`` with the actual attribute name +you want to use. + +In the case of {}-formatting, you can specify formatting flags by placing them +after the attribute name, separated from it with a colon. For example: a +placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as +``004``. Refer to the :meth:`str.format` documentation for full details on +the options available to you. + ++----------------+-------------------------+-----------------------------------------------+ +| Attribute name | Format | Description | ++================+=========================+===============================================+ +| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | +| | format this yourself. | produce ``message``, or a dict whose values | +| | | are used for the merge (when there is only one| +| | | argument, and it is a dictionary). | ++----------------+-------------------------+-----------------------------------------------+ +| asctime | ``%(asctime)s`` | Human-readable time when the | +| | | :class:`LogRecord` was created. By default | +| | | this is of the form '2003-07-08 16:49:45,896' | +| | | (the numbers after the comma are millisecond | +| | | portion of the time). | ++----------------+-------------------------+-----------------------------------------------+ +| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | +| | | (as returned by :func:`time.time`). | ++----------------+-------------------------+-----------------------------------------------+ +| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | +| | format this yourself. | if no exception has occurred, ``None``. | ++----------------+-------------------------+-----------------------------------------------+ +| filename | ``%(filename)s`` | Filename portion of ``pathname``. | ++----------------+-------------------------+-----------------------------------------------+ +| funcName | ``%(funcName)s`` | Name of function containing the logging call. | ++----------------+-------------------------+-----------------------------------------------+ +| levelname | ``%(levelname)s`` | Text logging level for the message | +| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | +| | | ``'ERROR'``, ``'CRITICAL'``). | ++----------------+-------------------------+-----------------------------------------------+ +| levelno | ``%(levelno)s`` | Numeric logging level for the message | +| | | (:const:`DEBUG`, :const:`INFO`, | +| | | :const:`WARNING`, :const:`ERROR`, | +| | | :const:`CRITICAL`). | ++----------------+-------------------------+-----------------------------------------------+ +| lineno | ``%(lineno)d`` | Source line number where the logging call was | +| | | issued (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| message | ``%(message)s`` | The logged message, computed as ``msg % | +| | | args``. This is set when | +| | | :meth:`Formatter.format` is invoked. | ++----------------+-------------------------+-----------------------------------------------+ +| module | ``%(module)s`` | Module (name portion of ``filename``). | ++----------------+-------------------------+-----------------------------------------------+ +| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | +| | | :class:`LogRecord` was created. | ++----------------+-------------------------+-----------------------------------------------+ +| msg | You shouldn't need to | The format string passed in the original | +| | format this yourself. | logging call. Merged with ``args`` to | +| | | produce ``message``, or an arbitrary object | +| | | (see :ref:`arbitrary-object-messages`). | ++----------------+-------------------------+-----------------------------------------------+ +| name | ``%(name)s`` | Name of the logger used to log the call. | ++----------------+-------------------------+-----------------------------------------------+ +| pathname | ``%(pathname)s`` | Full pathname of the source file where the | +| | | logging call was issued (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| process | ``%(process)d`` | Process ID (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| processName | ``%(processName)s`` | Process name (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | +| | | created, relative to the time the logging | +| | | module was loaded. | ++----------------+-------------------------+-----------------------------------------------+ +| stack_info | You shouldn't need to | Stack frame information (where available) | +| | format this yourself. | from the bottom of the stack in the current | +| | | thread, up to and including the stack frame | +| | | of the logging call which resulted in the | +| | | creation of this record. | ++----------------+-------------------------+-----------------------------------------------+ +| thread | ``%(thread)d`` | Thread ID (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| threadName | ``%(threadName)s`` | Thread name (if available). | ++----------------+-------------------------+-----------------------------------------------+ + +.. versionchanged:: 3.1 + *processName* was added. + + +.. _logger-adapter: + +LoggerAdapter Objects +--------------------- + +:class:`LoggerAdapter` instances are used to conveniently pass contextual +information into logging calls. For a usage example, see the section on +:ref:`adding contextual information to your logging output `. + +.. class:: LoggerAdapter(logger, extra) + + Returns an instance of :class:`LoggerAdapter` initialized with an + underlying :class:`Logger` instance and a dict-like object. + + .. method:: process(msg, kwargs) + + Modifies the message and/or keyword arguments passed to a logging call in + order to insert contextual information. This implementation takes the object + passed as *extra* to the constructor and adds it to *kwargs* using key + 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the + (possibly modified) versions of the arguments passed in. + +In addition to the above, :class:`LoggerAdapter` supports the following +methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, +:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, +:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, +:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and +:meth:`~Logger.hasHandlers`. These methods have the same signatures as their +counterparts in :class:`Logger`, so you can use the two types of instances +interchangeably. + +.. versionchanged:: 3.2 + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + +Thread Safety +------------- + +The logging module is intended to be thread-safe without any special work +needing to be done by its clients. It achieves this though using threading +locks; there is one lock to serialize access to the module's shared data, and +each handler also creates a lock to serialize access to its underlying I/O. + +If you are implementing asynchronous signal handlers using the :mod:`signal` +module, you may not be able to use logging from within such handlers. This is +because lock implementations in the :mod:`threading` module are not always +re-entrant, and so cannot be invoked from such signal handlers. + + +Module-Level Functions +---------------------- + +In addition to the classes described above, there are a number of module-level +functions. + + +.. function:: getLogger(name=None) + + Return a logger with the specified name or, if name is ``None``, return a + logger which is the root logger of the hierarchy. If specified, the name is + typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. + Choice of these names is entirely up to the developer who is using logging. + + All calls to this function with a given name return the same logger instance. + This means that logger instances never need to be passed between different parts + of an application. + + +.. function:: getLoggerClass() + + Return either the standard :class:`Logger` class, or the last class passed to + :func:`setLoggerClass`. This function may be called from within a new class + definition, to ensure that installing a customized :class:`Logger` class will + not undo customizations already applied by other code. For example:: + + class MyLogger(logging.getLoggerClass()): + # ... override behaviour here + + +.. function:: getLogRecordFactory() + + Return a callable which is used to create a :class:`LogRecord`. + + .. versionadded:: 3.2 + This function has been provided, along with :func:`setLogRecordFactory`, + to allow developers more control over how the :class:`LogRecord` + representing a logging event is constructed. + + See :func:`setLogRecordFactory` for more information about the how the + factory is called. + +.. function:: debug(msg, *args, **kwargs) + + Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the + message format string, and the *args* are the arguments which are merged into + *msg* using the string formatting operator. (Note that this means that you can + use keywords in the format string, together with a single dictionary argument.) + + There are three keyword arguments in *kwargs* which are inspected: *exc_info* + which, if it does not evaluate as false, causes exception information to be + added to the logging message. If an exception tuple (in the format returned by + :func:`sys.exc_info`) or an exception instance is provided, it is used; + otherwise, :func:`sys.exc_info` is called to get the exception information. + + The second optional keyword argument is *stack_info*, which defaults to + ``False``. If true, stack information is added to the logging + message, including the actual logging call. Note that this is not the same + stack information as that displayed through specifying *exc_info*: The + former is stack frames from the bottom of the stack up to the logging call + in the current thread, whereas the latter is information about stack frames + which have been unwound, following an exception, while searching for + exception handlers. + + You can specify *stack_info* independently of *exc_info*, e.g. to just show + how you got to a certain point in your code, even when no exceptions were + raised. The stack frames are printed following a header line which says: + + .. code-block:: none + + Stack (most recent call last): + + This mimics the ``Traceback (most recent call last):`` which is used when + displaying exception frames. + + The third optional keyword argument is *extra* which can be used to pass a + dictionary which is used to populate the __dict__ of the LogRecord created for + the logging event with user-defined attributes. These custom attributes can then + be used as you like. For example, they could be incorporated into logged + messages. For example:: + + FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + logging.basicConfig(format=FORMAT) + d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} + logging.warning('Protocol problem: %s', 'connection reset', extra=d) + + would print something like: + + .. code-block:: none + + 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset + + The keys in the dictionary passed in *extra* should not clash with the keys used + by the logging system. (See the :class:`Formatter` documentation for more + information on which keys are used by the logging system.) + + If you choose to use these attributes in logged messages, you need to exercise + some care. In the above example, for instance, the :class:`Formatter` has been + set up with a format string which expects 'clientip' and 'user' in the attribute + dictionary of the LogRecord. If these are missing, the message will not be + logged because a string formatting exception will occur. So in this case, you + always need to pass the *extra* dictionary with these keys. + + While this might be annoying, this feature is intended for use in specialized + circumstances, such as multi-threaded servers where the same code executes in + many contexts, and interesting conditions which arise are dependent on this + context (such as remote client IP address and authenticated user name, in the + above example). In such circumstances, it is likely that specialized + :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + + .. versionchanged:: 3.2 + The *stack_info* parameter was added. + +.. function:: info(msg, *args, **kwargs) + + Logs a message with level :const:`INFO` on the root logger. The arguments are + interpreted as for :func:`debug`. + + +.. function:: warning(msg, *args, **kwargs) + + Logs a message with level :const:`WARNING` on the root logger. The arguments + are interpreted as for :func:`debug`. + + .. note:: There is an obsolete function ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + + +.. function:: error(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on the root logger. The arguments are + interpreted as for :func:`debug`. + + +.. function:: critical(msg, *args, **kwargs) + + Logs a message with level :const:`CRITICAL` on the root logger. The arguments + are interpreted as for :func:`debug`. + + +.. function:: exception(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on the root logger. The arguments are + interpreted as for :func:`debug`. Exception info is added to the logging + message. This function should only be called from an exception handler. + +.. function:: log(level, msg, *args, **kwargs) + + Logs a message with level *level* on the root logger. The other arguments are + interpreted as for :func:`debug`. + + .. note:: The above module-level convenience functions, which delegate to the + root logger, call :func:`basicConfig` to ensure that at least one handler + is available. Because of this, they should *not* be used in threads, + in versions of Python earlier than 2.7.1 and 3.2, unless at least one + handler has been added to the root logger *before* the threads are + started. In earlier versions of Python, due to a thread safety shortcoming + in :func:`basicConfig`, this can (under rare circumstances) lead to + handlers being added multiple times to the root logger, which can in turn + lead to multiple messages for the same event. + +.. function:: disable(level=CRITICAL) + + Provides an overriding level *level* for all loggers which takes precedence over + the logger's own level. When the need arises to temporarily throttle logging + output down across the whole application, this function can be useful. Its + effect is to disable all logging calls of severity *level* and below, so that + if you call it with a value of INFO, then all INFO and DEBUG events would be + discarded, whereas those of severity WARNING and above would be processed + according to the logger's effective level. If + ``logging.disable(logging.NOTSET)`` is called, it effectively removes this + overriding level, so that logging output again depends on the effective + levels of individual loggers. + + Note that if you have defined any custom logging level higher than + ``CRITICAL`` (this is not recommended), you won't be able to rely on the + default value for the *level* parameter, but will have to explicitly supply a + suitable value. + + .. versionchanged:: 3.7 + The *level* parameter was defaulted to level ``CRITICAL``. See + :issue:`28524` for more information about this change. + +.. function:: addLevelName(level, levelName) + + Associates level *level* with text *levelName* in an internal dictionary, which is + used to map numeric levels to a textual representation, for example when a + :class:`Formatter` formats a message. This function can also be used to define + your own levels. The only constraints are that all levels used must be + registered using this function, levels should be positive integers and they + should increase in increasing order of severity. + + .. note:: If you are thinking of defining your own levels, please see the + section on :ref:`custom-levels`. + +.. function:: getLevelName(level) + + Returns the textual representation of logging level *level*. If the level is one + of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, + :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you + have associated levels with names using :func:`addLevelName` then the name you + have associated with *level* is returned. If a numeric value corresponding to one + of the defined levels is passed in, the corresponding string representation is + returned. Otherwise, the string 'Level %s' % level is returned. + + .. note:: Levels are internally integers (as they need to be compared in the + logging logic). This function is used to convert between an integer level + and the level name displayed in the formatted log output by means of the + ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). + + .. versionchanged:: 3.4 + In Python versions earlier than 3.4, this function could also be passed a + text level, and would return the corresponding numeric value of the level. + This undocumented behaviour was considered a mistake, and was removed in + Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. + +.. function:: makeLogRecord(attrdict) + + Creates and returns a new :class:`LogRecord` instance whose attributes are + defined by *attrdict*. This function is useful for taking a pickled + :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting + it as a :class:`LogRecord` instance at the receiving end. + + +.. function:: basicConfig(**kwargs) + + Does basic configuration for the logging system by creating a + :class:`StreamHandler` with a default :class:`Formatter` and adding it to the + root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, + :func:`error` and :func:`critical` will call :func:`basicConfig` automatically + if no handlers are defined for the root logger. + + This function does nothing if the root logger already has handlers + configured, unless the keyword argument *force* is set to ``True``. + + .. note:: This function should be called from the main thread + before other threads are started. In versions of Python prior to + 2.7.1 and 3.2, if this function is called from multiple threads, + it is possible (in rare circumstances) that a handler will be added + to the root logger more than once, leading to unexpected results + such as messages being duplicated in the log. + + The following keyword arguments are supported. + + .. tabularcolumns:: |l|L| + + +--------------+---------------------------------------------+ + | Format | Description | + +==============+=============================================+ + | *filename* | Specifies that a :class:`FileHandler` be | + | | created, using the specified filename, | + | | rather than a :class:`StreamHandler`. | + +--------------+---------------------------------------------+ + | *filemode* | If *filename* is specified, open the file | + | | in this :ref:`mode `. Defaults | + | | to ``'a'``. | + +--------------+---------------------------------------------+ + | *format* | Use the specified format string for the | + | | handler. Defaults to attributes | + | | ``levelname``, ``name`` and ``message`` | + | | separated by colons. | + +--------------+---------------------------------------------+ + | *datefmt* | Use the specified date/time format, as | + | | accepted by :func:`time.strftime`. | + +--------------+---------------------------------------------+ + | *style* | If *format* is specified, use this style | + | | for the format string. One of ``'%'``, | + | | ``'{'`` or ``'$'`` for :ref:`printf-style | + | | `, | + | | :meth:`str.format` or | + | | :class:`string.Template` respectively. | + | | Defaults to ``'%'``. | + +--------------+---------------------------------------------+ + | *level* | Set the root logger level to the specified | + | | :ref:`level `. | + +--------------+---------------------------------------------+ + | *stream* | Use the specified stream to initialize the | + | | :class:`StreamHandler`. Note that this | + | | argument is incompatible with *filename* - | + | | if both are present, a ``ValueError`` is | + | | raised. | + +--------------+---------------------------------------------+ + | *handlers* | If specified, this should be an iterable of | + | | already created handlers to add to the root | + | | logger. Any handlers which don't already | + | | have a formatter set will be assigned the | + | | default formatter created in this function. | + | | Note that this argument is incompatible | + | | with *filename* or *stream* - if both | + | | are present, a ``ValueError`` is raised. | + +--------------+---------------------------------------------+ + | *force* | If this keyword argument is specified as | + | | true, any existing handlers attached to the | + | | root logger are removed and closed, before | + | | carrying out the configuration as specified | + | | by the other arguments. | + +--------------+---------------------------------------------+ + | *encoding* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. | + +--------------+---------------------------------------------+ + | *errors* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. If not | + | | specified, the value 'backslashreplace' is | + | | used. Note that if ``None`` is specified, | + | | it will be passed as such to :func:`open`, | + | | which means that it will be treated the | + | | same as passing 'errors'. | + +--------------+---------------------------------------------+ + + .. versionchanged:: 3.2 + The *style* argument was added. + + .. versionchanged:: 3.3 + The *handlers* argument was added. Additional checks were added to + catch situations where incompatible arguments are specified (e.g. + *handlers* together with *stream* or *filename*, or *stream* + together with *filename*). + + .. versionchanged:: 3.8 + The *force* argument was added. + + .. versionchanged:: 3.9 + The *encoding* and *errors* arguments were added. + +.. function:: shutdown() + + Informs the logging system to perform an orderly shutdown by flushing and + closing all handlers. This should be called at application exit and no + further use of the logging system should be made after this call. + + When the logging module is imported, it registers this function as an exit + handler (see :mod:`atexit`), so normally there's no need to do that + manually. + + +.. function:: setLoggerClass(klass) + + Tells the logging system to use the class *klass* when instantiating a logger. + The class should define :meth:`__init__` such that only a name argument is + required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + function is typically called before any loggers are instantiated by applications + which need to use custom logger behavior. After this call, as at any other + time, do not instantiate loggers directly using the subclass: continue to use + the :func:`logging.getLogger` API to get your loggers. + + +.. function:: setLogRecordFactory(factory) + + Set a callable which is used to create a :class:`LogRecord`. + + :param factory: The factory callable to be used to instantiate a log record. + + .. versionadded:: 3.2 + This function has been provided, along with :func:`getLogRecordFactory`, to + allow developers more control over how the :class:`LogRecord` representing + a logging event is constructed. + + The factory has the following signature: + + ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` + + :name: The logger name. + :level: The logging level (numeric). + :fn: The full pathname of the file where the logging call was made. + :lno: The line number in the file where the logging call was made. + :msg: The logging message. + :args: The arguments for the logging message. + :exc_info: An exception tuple, or ``None``. + :func: The name of the function or method which invoked the logging + call. + :sinfo: A stack traceback such as is provided by + :func:`traceback.print_stack`, showing the call hierarchy. + :kwargs: Additional keyword arguments. + + +Module-Level Attributes +----------------------- + +.. attribute:: lastResort + + A "handler of last resort" is available through this attribute. This + is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of + ``WARNING``, and is used to handle logging events in the absence of any + logging configuration. The end result is to just print the message to + ``sys.stderr``. This replaces the earlier error message saying that + "no handlers could be found for logger XYZ". If you need the earlier + behaviour for some reason, ``lastResort`` can be set to ``None``. + + .. versionadded:: 3.2 + +Integration with the warnings module +------------------------------------ + +The :func:`captureWarnings` function can be used to integrate :mod:`logging` +with the :mod:`warnings` module. + +.. function:: captureWarnings(capture) + + This function is used to turn the capture of warnings by logging on and + off. + + If *capture* is ``True``, warnings issued by the :mod:`warnings` module will + be redirected to the logging system. Specifically, a warning will be + formatted using :func:`warnings.formatwarning` and the resulting string + logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. + + If *capture* is ``False``, the redirection of warnings to the logging system + will stop, and warnings will be redirected to their original destinations + (i.e. those in effect before ``captureWarnings(True)`` was called). + + +.. seealso:: + + Module :mod:`logging.config` + Configuration API for the logging module. + + Module :mod:`logging.handlers` + Useful handlers included with the logging module. + + :pep:`282` - A Logging System + The proposal which described this feature for inclusion in the Python standard + library. + + `Original Python logging package `_ + This is the original source for the :mod:`logging` package. The version of the + package available from this site is suitable for use with Python 1.5.2, 2.1.x + and 2.2.x, which do not include the :mod:`logging` package in the standard + library. diff --git a/Doc/library/readline.rst.bak b/Doc/library/readline.rst.bak new file mode 100644 index 000000000000000..3ff64885f7fced7 --- /dev/null +++ b/Doc/library/readline.rst.bak @@ -0,0 +1,361 @@ +:mod:`readline` --- GNU readline interface +========================================== + +.. module:: readline + :platform: Unix + :synopsis: GNU readline support for Python. + +.. sectionauthor:: Skip Montanaro + +-------------- + +The :mod:`readline` module defines a number of functions to facilitate +completion and reading/writing of history files from the Python interpreter. +This module can be used directly, or via the :mod:`rlcompleter` module, which +supports completion of Python identifiers at the interactive prompt. Settings +made using this module affect the behaviour of both the interpreter's +interactive prompt and the prompts offered by the built-in :func:`input` +function. + +Readline keybindings may be configured via an initialization file, typically +``.inputrc`` in your home directory. See `Readline Init File +`_ +in the GNU Readline manual for information about the format and +allowable constructs of that file, and the capabilities of the +Readline library in general. + +.. note:: + + The underlying Readline library API may be implemented by + the ``libedit`` library instead of GNU readline. + On macOS the :mod:`readline` module detects which library is being used + at run time. + + The configuration file for ``libedit`` is different from that + of GNU readline. If you programmatically load configuration strings + you can check for the text "libedit" in :const:`readline.__doc__` + to differentiate between GNU readline and libedit. + + If you use *editline*/``libedit`` readline emulation on macOS, the + initialization file located in your home directory is named + ``.editrc``. For example, the following content in ``~/.editrc`` will + turn ON *vi* keybindings and TAB completion:: + + python:bind -v + python:bind ^I rl_complete + + +Init file +--------- + +The following functions relate to the init file and user configuration: + + +.. function:: parse_and_bind(string) + + Execute the init line provided in the *string* argument. This calls + :c:func:`rl_parse_and_bind` in the underlying library. + + +.. function:: read_init_file([filename]) + + Execute a readline initialization file. The default filename is the last filename + used. This calls :c:func:`rl_read_init_file` in the underlying library. + + +Line buffer +----------- + +The following functions operate on the line buffer: + + +.. function:: get_line_buffer() + + Return the current contents of the line buffer (:c:data:`rl_line_buffer` + in the underlying library). + + +.. function:: insert_text(string) + + Insert text into the line buffer at the cursor position. This calls + :c:func:`rl_insert_text` in the underlying library, but ignores + the return value. + + +.. function:: redisplay() + + Change what's displayed on the screen to reflect the current contents of the + line buffer. This calls :c:func:`rl_redisplay` in the underlying library. + + +History file +------------ + +The following functions operate on a history file: + + +.. function:: read_history_file([filename]) + + Load a readline history file, and append it to the history list. + The default filename is :file:`~/.history`. This calls + :c:func:`read_history` in the underlying library. + + +.. function:: write_history_file([filename]) + + Save the history list to a readline history file, overwriting any + existing file. The default filename is :file:`~/.history`. This calls + :c:func:`write_history` in the underlying library. + + +.. function:: append_history_file(nelements[, filename]) + + Append the last *nelements* items of history to a file. The default filename is + :file:`~/.history`. The file must already exist. This calls + :c:func:`append_history` in the underlying library. This function + only exists if Python was compiled for a version of the library + that supports it. + + .. versionadded:: 3.5 + + +.. function:: get_history_length() + set_history_length(length) + + Set or return the desired number of lines to save in the history file. + The :func:`write_history_file` function uses this value to truncate + the history file, by calling :c:func:`history_truncate_file` in + the underlying library. Negative values imply + unlimited history file size. + + +History list +------------ + +The following functions operate on a global history list: + + +.. function:: clear_history() + + Clear the current history. This calls :c:func:`clear_history` in the + underlying library. The Python function only exists if Python was + compiled for a version of the library that supports it. + + +.. function:: get_current_history_length() + + Return the number of items currently in the history. (This is different from + :func:`get_history_length`, which returns the maximum number of lines that will + be written to a history file.) + + +.. function:: get_history_item(index) + + Return the current contents of history item at *index*. The item index + is one-based. This calls :c:func:`history_get` in the underlying library. + + +.. function:: remove_history_item(pos) + + Remove history item specified by its position from the history. + The position is zero-based. This calls :c:func:`remove_history` in + the underlying library. + + +.. function:: replace_history_item(pos, line) + + Replace history item specified by its position with *line*. + The position is zero-based. This calls :c:func:`replace_history_entry` + in the underlying library. + + +.. function:: add_history(line) + + Append *line* to the history buffer, as if it was the last line typed. + This calls :c:func:`add_history` in the underlying library. + + +.. function:: set_auto_history(enabled) + + Enable or disable automatic calls to :c:func:`add_history` when reading + input via readline. The *enabled* argument should be a Boolean value + that when true, enables auto history, and that when false, disables + auto history. + + .. versionadded:: 3.6 + + .. impl-detail:: + Auto history is enabled by default, and changes to this do not persist + across multiple sessions. + + +Startup hooks +------------- + + +.. function:: set_startup_hook([function]) + + Set or remove the function invoked by the :c:data:`rl_startup_hook` + callback of the underlying library. If *function* is specified, it will + be used as the new hook function; if omitted or ``None``, any function + already installed is removed. The hook is called with no + arguments just before readline prints the first prompt. + + +.. function:: set_pre_input_hook([function]) + + Set or remove the function invoked by the :c:data:`rl_pre_input_hook` + callback of the underlying library. If *function* is specified, it will + be used as the new hook function; if omitted or ``None``, any + function already installed is removed. The hook is called + with no arguments after the first prompt has been printed and just before + readline starts reading input characters. This function only exists + if Python was compiled for a version of the library that supports it. + + +Completion +---------- + +The following functions relate to implementing a custom word completion +function. This is typically operated by the Tab key, and can suggest and +automatically complete a word being typed. By default, Readline is set up +to be used by :mod:`rlcompleter` to complete Python identifiers for +the interactive interpreter. If the :mod:`readline` module is to be used +with a custom completer, a different set of word delimiters should be set. + + +.. function:: set_completer([function]) + + Set or remove the completer function. If *function* is specified, it will be + used as the new completer function; if omitted or ``None``, any completer + function already installed is removed. The completer function is called as + ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it + returns a non-string value. It should return the next possible completion + starting with *text*. + + The installed completer function is invoked by the *entry_func* callback + passed to :c:func:`rl_completion_matches` in the underlying library. + The *text* string comes from the first parameter to the + :c:data:`rl_attempted_completion_function` callback of the + underlying library. + + +.. function:: get_completer() + + Get the completer function, or ``None`` if no completer function has been set. + + +.. function:: get_completion_type() + + Get the type of completion being attempted. This returns the + :c:data:`rl_completion_type` variable in the underlying library as + an integer. + + +.. function:: get_begidx() + get_endidx() + + Get the beginning or ending index of the completion scope. + These indexes are the *start* and *end* arguments passed to the + :c:data:`rl_attempted_completion_function` callback of the + underlying library. The values may be different in the same + input editing scenario based on the underlying C readline implemtation. + Ex: libedit is known to behave differently than libreadline. + + +.. function:: set_completer_delims(string) + get_completer_delims() + + Set or get the word delimiters for completion. These determine the + start of the word to be considered for completion (the completion scope). + These functions access the :c:data:`rl_completer_word_break_characters` + variable in the underlying library. + + +.. function:: set_completion_display_matches_hook([function]) + + Set or remove the completion display function. If *function* is + specified, it will be used as the new completion display function; + if omitted or ``None``, any completion display function already + installed is removed. This sets or clears the + :c:data:`rl_completion_display_matches_hook` callback in the + underlying library. The completion display function is called as + ``function(substitution, [matches], longest_match_length)`` once + each time matches need to be displayed. + + +.. _readline-example: + +Example +------- + +The following example demonstrates how to use the :mod:`readline` module's +history reading and writing functions to automatically load and save a history +file named :file:`.python_history` from the user's home directory. The code +below would normally be executed automatically during interactive sessions +from the user's :envvar:`PYTHONSTARTUP` file. :: + + import atexit + import os + import readline + + histfile = os.path.join(os.path.expanduser("~"), ".python_history") + try: + readline.read_history_file(histfile) + # default history len is -1 (infinite), which may grow unruly + readline.set_history_length(1000) + except FileNotFoundError: + pass + + atexit.register(readline.write_history_file, histfile) + +This code is actually automatically run when Python is run in +:ref:`interactive mode ` (see :ref:`rlcompleter-config`). + +The following example achieves the same goal but supports concurrent interactive +sessions, by only appending the new history. :: + + import atexit + import os + import readline + histfile = os.path.join(os.path.expanduser("~"), ".python_history") + + try: + readline.read_history_file(histfile) + h_len = readline.get_current_history_length() + except FileNotFoundError: + open(histfile, 'wb').close() + h_len = 0 + + def save(prev_h_len, histfile): + new_h_len = readline.get_current_history_length() + readline.set_history_length(1000) + readline.append_history_file(new_h_len - prev_h_len, histfile) + atexit.register(save, h_len, histfile) + +The following example extends the :class:`code.InteractiveConsole` class to +support history save/restore. :: + + import atexit + import code + import os + import readline + + class HistoryConsole(code.InteractiveConsole): + def __init__(self, locals=None, filename="", + histfile=os.path.expanduser("~/.console-history")): + code.InteractiveConsole.__init__(self, locals, filename) + self.init_history(histfile) + + def init_history(self, histfile): + readline.parse_and_bind("tab: complete") + if hasattr(readline, "read_history_file"): + try: + readline.read_history_file(histfile) + except FileNotFoundError: + pass + atexit.register(self.save_history, histfile) + + def save_history(self, histfile): + readline.set_history_length(1000) + readline.write_history_file(histfile) diff --git a/Doc/library/sqlite3.rst.bak b/Doc/library/sqlite3.rst.bak new file mode 100644 index 000000000000000..172ce6c6bb03ba3 --- /dev/null +++ b/Doc/library/sqlite3.rst.bak @@ -0,0 +1,1094 @@ +:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================ + +.. module:: sqlite3 + :synopsis: A DB-API 2.0 implementation using SQLite 3.x. + +.. sectionauthor:: Gerhard Häring + +**Source code:** :source:`Lib/sqlite3/` + +-------------- + +SQLite is a C library that provides a lightweight disk-based database that +doesn't require a separate server process and allows accessing the database +using a nonstandard variant of the SQL query language. Some applications can use +SQLite for internal data storage. It's also possible to prototype an +application using SQLite and then port the code to a larger database such as +PostgreSQL or Oracle. + +The sqlite3 module was written by Gerhard Häring. It provides a SQL interface +compliant with the DB-API 2.0 specification described by :pep:`249`, and +requires SQLite 3.7.15 or newer. + +To use the module, you must first create a :class:`Connection` object that +represents the database. Here the data will be stored in the +:file:`example.db` file:: + + import sqlite3 + con = sqlite3.connect('example.db') + +You can also supply the special name ``:memory:`` to create a database in RAM. + +Once you have a :class:`Connection`, you can create a :class:`Cursor` object +and call its :meth:`~Cursor.execute` method to perform SQL commands:: + + cur = con.cursor() + + # Create table + cur.execute('''CREATE TABLE stocks + (date text, trans text, symbol text, qty real, price real)''') + + # Insert a row of data + cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") + + # Save (commit) the changes + con.commit() + + # We can also close the connection if we are done with it. + # Just be sure any changes have been committed or they will be lost. + con.close() + +The data you've saved is persistent and is available in subsequent sessions:: + + import sqlite3 + con = sqlite3.connect('example.db') + cur = con.cursor() + +Usually your SQL operations will need to use values from Python variables. You +shouldn't assemble your query using Python's string operations because doing so +is insecure; it makes your program vulnerable to an SQL injection attack +(see https://xkcd.com/327/ for humorous example of what can go wrong). + +Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder +wherever you want to use a value, and then provide a tuple of values as the +second argument to the cursor's :meth:`~Cursor.execute` method. (Other database +modules may use a different placeholder, such as ``%s`` or ``:1``.) For +example:: + + # Never do this -- insecure! + symbol = 'RHAT' + cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) + + # Do this instead + t = ('RHAT',) + cur.execute('SELECT * FROM stocks WHERE symbol=?', t) + print(cur.fetchone()) + + # Larger example that inserts many records at a time + purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), + ('2006-04-06', 'SELL', 'IBM', 500, 53.00), + ] + cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) + +To retrieve data after executing a SELECT statement, you can either treat the +cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to +retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the +matching rows. + +This example uses the iterator form:: + + >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): + print(row) + + ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) + ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) + ('2006-04-06', 'SELL', 'IBM', 500, 53.0) + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) + + +.. seealso:: + + https://www.sqlite.org + The SQLite web page; the documentation describes the syntax and the + available data types for the supported SQL dialect. + + https://www.w3schools.com/sql/ + Tutorial, reference and examples for learning SQL syntax. + + :pep:`249` - Database API Specification 2.0 + PEP written by Marc-André Lemburg. + + +.. _sqlite3-module-contents: + +Module functions and constants +------------------------------ + + +.. data:: version + + The version number of this module, as a string. This is not the version of + the SQLite library. + + +.. data:: version_info + + The version number of this module, as a tuple of integers. This is not the + version of the SQLite library. + + +.. data:: sqlite_version + + The version number of the run-time SQLite library, as a string. + + +.. data:: sqlite_version_info + + The version number of the run-time SQLite library, as a tuple of integers. + + +.. data:: PARSE_DECLTYPES + + This constant is meant to be used with the *detect_types* parameter of the + :func:`connect` function. + + Setting it makes the :mod:`sqlite3` module parse the declared type for each + column it returns. It will parse out the first word of the declared type, + i. e. for "integer primary key", it will parse out "integer", or for + "number(10)" it will parse out "number". Then for that column, it will look + into the converters dictionary and use the converter function registered for + that type there. + + +.. data:: PARSE_COLNAMES + + This constant is meant to be used with the *detect_types* parameter of the + :func:`connect` function. + + Setting this makes the SQLite interface parse the column name for each column it + returns. It will look for a string formed [mytype] in there, and then decide + that 'mytype' is the type of the column. It will try to find an entry of + 'mytype' in the converters dictionary and then use the converter function found + there to return the value. The column name found in :attr:`Cursor.description` + does not include the type, i. e. if you use something like + ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out + everything until the first ``'['`` for the column name and strip + the preceeding space: the column name would simply be "Expiration date". + + +.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) + + Opens a connection to the SQLite database file *database*. By default returns a + :class:`Connection` object, unless a custom *factory* is given. + + *database* is a :term:`path-like object` giving the pathname (absolute or + relative to the current working directory) of the database file to be opened. + You can use ``":memory:"`` to open a database connection to a database that + resides in RAM instead of on disk. + + When a database is accessed by multiple connections, and one of the processes + modifies the database, the SQLite database is locked until that transaction is + committed. The *timeout* parameter specifies how long the connection should wait + for the lock to go away until raising an exception. The default for the timeout + parameter is 5.0 (five seconds). + + For the *isolation_level* parameter, please see the + :attr:`~Connection.isolation_level` property of :class:`Connection` objects. + + SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If + you want to use other types you must add support for them yourself. The + *detect_types* parameter and the using custom **converters** registered with the + module-level :func:`register_converter` function allow you to easily do that. + + *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to + any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn + type detection on. Due to SQLite behaviour, types can't be detected for generated + fields (for example ``max(data)``), even when *detect_types* parameter is set. In + such case, the returned type is :class:`str`. + + By default, *check_same_thread* is :const:`True` and only the creating thread may + use the connection. If set :const:`False`, the returned connection may be shared + across multiple threads. When using multiple threads with the same connection + writing operations should be serialized by the user to avoid data corruption. + + By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the + connect call. You can, however, subclass the :class:`Connection` class and make + :func:`connect` use your class instead by providing your class for the *factory* + parameter. + + Consult the section :ref:`sqlite3-types` of this manual for details. + + The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing + overhead. If you want to explicitly set the number of statements that are cached + for the connection, you can set the *cached_statements* parameter. The currently + implemented default is to cache 100 statements. + + If *uri* is true, *database* is interpreted as a URI. This allows you + to specify options. For example, to open a database in read-only mode + you can use:: + + db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) + + More information about this feature, including a list of recognized options, can + be found in the `SQLite URI documentation `_. + + .. audit-event:: sqlite3.connect database sqlite3.connect + + .. versionchanged:: 3.4 + Added the *uri* parameter. + + .. versionchanged:: 3.7 + *database* can now also be a :term:`path-like object`, not only a string. + + +.. function:: register_converter(typename, callable) + + Registers a callable to convert a bytestring from the database into a custom + Python type. The callable will be invoked for all database values that are of + the type *typename*. Confer the parameter *detect_types* of the :func:`connect` + function for how the type detection works. Note that *typename* and the name of + the type in your query are matched in case-insensitive manner. + + +.. function:: register_adapter(type, callable) + + Registers a callable to convert the custom Python type *type* into one of + SQLite's supported types. The callable *callable* accepts as single parameter + the Python value, and must return a value of the following types: int, + float, str or bytes. + + +.. function:: complete_statement(sql) + + Returns :const:`True` if the string *sql* contains one or more complete SQL + statements terminated by semicolons. It does not verify that the SQL is + syntactically correct, only that there are no unclosed string literals and the + statement is terminated by a semicolon. + + This can be used to build a shell for SQLite, as in the following example: + + + .. literalinclude:: ../includes/sqlite3/complete_statement.py + + +.. function:: enable_callback_tracebacks(flag) + + By default you will not get any tracebacks in user-defined functions, + aggregates, converters, authorizer callbacks etc. If you want to debug them, + you can call this function with *flag* set to ``True``. Afterwards, you will + get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to + disable the feature again. + + +.. _sqlite3-connection-objects: + +Connection Objects +------------------ + +.. class:: Connection + + A SQLite database connection has the following attributes and methods: + + .. attribute:: isolation_level + + Get or set the current default isolation level. :const:`None` for autocommit mode or + one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section + :ref:`sqlite3-controlling-transactions` for a more detailed explanation. + + .. attribute:: in_transaction + + :const:`True` if a transaction is active (there are uncommitted changes), + :const:`False` otherwise. Read-only attribute. + + .. versionadded:: 3.2 + + .. method:: cursor(factory=Cursor) + + The cursor method accepts a single optional parameter *factory*. If + supplied, this must be a callable returning an instance of :class:`Cursor` + or its subclasses. + + .. method:: commit() + + This method commits the current transaction. If you don't call this method, + anything you did since the last call to ``commit()`` is not visible from + other database connections. If you wonder why you don't see the data you've + written to the database, please check you didn't forget to call this method. + + .. method:: rollback() + + This method rolls back any changes to the database since the last call to + :meth:`commit`. + + .. method:: close() + + This closes the database connection. Note that this does not automatically + call :meth:`commit`. If you just close your database connection without + calling :meth:`commit` first, your changes will be lost! + + .. method:: execute(sql[, parameters]) + + This is a nonstandard shortcut that creates a cursor object by calling + the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.execute` method with the *parameters* given, and returns + the cursor. + + .. method:: executemany(sql[, parameters]) + + This is a nonstandard shortcut that creates a cursor object by + calling the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.executemany` method with the *parameters* given, and + returns the cursor. + + .. method:: executescript(sql_script) + + This is a nonstandard shortcut that creates a cursor object by + calling the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.executescript` method with the given *sql_script*, and + returns the cursor. + + .. method:: create_function(name, num_params, func, *, deterministic=False) + + Creates a user-defined function that you can later use from within SQL + statements under the function name *name*. *num_params* is the number of + parameters the function accepts (if *num_params* is -1, the function may + take any number of arguments), and *func* is a Python callable that is + called as the SQL function. If *deterministic* is true, the created function + is marked as `deterministic `_, which + allows SQLite to perform additional optimizations. This flag is supported by + SQLite 3.8.3 or higher, :exc:`NotSupportedError` will be raised if used + with older versions. + + The function can return any of the types supported by SQLite: bytes, str, int, + float and ``None``. + + .. versionchanged:: 3.8 + The *deterministic* parameter was added. + + Example: + + .. literalinclude:: ../includes/sqlite3/md5func.py + + + .. method:: create_aggregate(name, num_params, aggregate_class) + + Creates a user-defined aggregate function. + + The aggregate class must implement a ``step`` method, which accepts the number + of parameters *num_params* (if *num_params* is -1, the function may take + any number of arguments), and a ``finalize`` method which will return the + final result of the aggregate. + + The ``finalize`` method can return any of the types supported by SQLite: + bytes, str, int, float and ``None``. + + Example: + + .. literalinclude:: ../includes/sqlite3/mysumaggr.py + + + .. method:: create_collation(name, callable) + + Creates a collation with the specified *name* and *callable*. The callable will + be passed two string arguments. It should return -1 if the first is ordered + lower than the second, 0 if they are ordered equal and 1 if the first is ordered + higher than the second. Note that this controls sorting (ORDER BY in SQL) so + your comparisons don't affect other SQL operations. + + Note that the callable will get its parameters as Python bytestrings, which will + normally be encoded in UTF-8. + + The following example shows a custom collation that sorts "the wrong way": + + .. literalinclude:: ../includes/sqlite3/collation_reverse.py + + To remove a collation, call ``create_collation`` with ``None`` as callable:: + + con.create_collation("reverse", None) + + + .. method:: interrupt() + + You can call this method from a different thread to abort any queries that might + be executing on the connection. The query will then abort and the caller will + get an exception. + + + .. method:: set_authorizer(authorizer_callback) + + This routine registers a callback. The callback is invoked for each attempt to + access a column of a table in the database. The callback should return + :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL + statement should be aborted with an error and :const:`SQLITE_IGNORE` if the + column should be treated as a NULL value. These constants are available in the + :mod:`sqlite3` module. + + The first argument to the callback signifies what kind of operation is to be + authorized. The second and third argument will be arguments or :const:`None` + depending on the first argument. The 4th argument is the name of the database + ("main", "temp", etc.) if applicable. The 5th argument is the name of the + inner-most trigger or view that is responsible for the access attempt or + :const:`None` if this access attempt is directly from input SQL code. + + Please consult the SQLite documentation about the possible values for the first + argument and the meaning of the second and third argument depending on the first + one. All necessary constants are available in the :mod:`sqlite3` module. + + + .. method:: set_progress_handler(handler, n) + + This routine registers a callback. The callback is invoked for every *n* + instructions of the SQLite virtual machine. This is useful if you want to + get called from SQLite during long-running operations, for example to update + a GUI. + + If you want to clear any previously installed progress handler, call the + method with :const:`None` for *handler*. + + Returning a non-zero value from the handler function will terminate the + currently executing query and cause it to raise an :exc:`OperationalError` + exception. + + + .. method:: set_trace_callback(trace_callback) + + Registers *trace_callback* to be called for each SQL statement that is + actually executed by the SQLite backend. + + The only argument passed to the callback is the statement (as string) that + is being executed. The return value of the callback is ignored. Note that + the backend does not only run statements passed to the :meth:`Cursor.execute` + methods. Other sources include the transaction management of the Python + module and the execution of triggers defined in the current database. + + Passing :const:`None` as *trace_callback* will disable the trace callback. + + .. versionadded:: 3.3 + + + .. method:: enable_load_extension(enabled) + + This routine allows/disallows the SQLite engine to load SQLite extensions + from shared libraries. SQLite extensions can define new functions, + aggregates or whole new virtual table implementations. One well-known + extension is the fulltext-search extension distributed with SQLite. + + Loadable extensions are disabled by default. See [#f1]_. + + .. versionadded:: 3.2 + + .. literalinclude:: ../includes/sqlite3/load_extension.py + + .. method:: load_extension(path) + + This routine loads a SQLite extension from a shared library. You have to + enable extension loading with :meth:`enable_load_extension` before you can + use this routine. + + Loadable extensions are disabled by default. See [#f1]_. + + .. versionadded:: 3.2 + + .. attribute:: row_factory + + You can change this attribute to a callable that accepts the cursor and the + original row as a tuple and will return the real result row. This way, you can + implement more advanced ways of returning results, such as returning an object + that can also access columns by name. + + Example: + + .. literalinclude:: ../includes/sqlite3/row_factory.py + + If returning a tuple doesn't suffice and you want name-based access to + columns, you should consider setting :attr:`row_factory` to the + highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both + index-based and case-insensitive name-based access to columns with almost no + memory overhead. It will probably be better than your own custom + dictionary-based approach or even a db_row based solution. + + .. XXX what's a db_row-based solution? + + + .. attribute:: text_factory + + Using this attribute you can control what objects are returned for the ``TEXT`` + data type. By default, this attribute is set to :class:`str` and the + :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to + return bytestrings instead, you can set it to :class:`bytes`. + + You can also set it to any other callable that accepts a single bytestring + parameter and returns the resulting object. + + See the following example code for illustration: + + .. literalinclude:: ../includes/sqlite3/text_factory.py + + + .. attribute:: total_changes + + Returns the total number of database rows that have been modified, inserted, or + deleted since the database connection was opened. + + + .. method:: iterdump + + Returns an iterator to dump the database in an SQL text format. Useful when + saving an in-memory database for later restoration. This function provides + the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` + shell. + + Example:: + + # Convert file existing_db.db to SQL dump file dump.sql + import sqlite3 + + con = sqlite3.connect('existing_db.db') + with open('dump.sql', 'w') as f: + for line in con.iterdump(): + f.write('%s\n' % line) + con.close() + + + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + bck = sqlite3.connect('backup.db') + with bck: + con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + .. versionadded:: 3.7 + + +.. _sqlite3-cursor-objects: + +Cursor Objects +-------------- + +.. class:: Cursor + + A :class:`Cursor` instance has the following attributes and methods. + + .. index:: single: ? (question mark); in SQL statements + .. index:: single: : (colon); in SQL statements + + .. method:: execute(sql[, parameters]) + + Executes an SQL statement. The SQL statement may be parameterized (i. e. + placeholders instead of SQL literals). The :mod:`sqlite3` module supports two + kinds of placeholders: question marks (qmark style) and named placeholders + (named style). + + Here's an example of both styles: + + .. literalinclude:: ../includes/sqlite3/execute_1.py + + :meth:`execute` will only execute a single SQL statement. If you try to execute + more than one statement with it, it will raise a :exc:`.Warning`. Use + :meth:`executescript` if you want to execute multiple SQL statements with one + call. + + + .. method:: executemany(sql, seq_of_parameters) + + Executes an SQL command against all parameter sequences or mappings found in + the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows + using an :term:`iterator` yielding parameters instead of a sequence. + + .. literalinclude:: ../includes/sqlite3/executemany_1.py + + Here's a shorter example using a :term:`generator`: + + .. literalinclude:: ../includes/sqlite3/executemany_2.py + + + .. method:: executescript(sql_script) + + This is a nonstandard convenience method for executing multiple SQL statements + at once. It issues a ``COMMIT`` statement first, then executes the SQL script it + gets as a parameter. + + *sql_script* can be an instance of :class:`str`. + + Example: + + .. literalinclude:: ../includes/sqlite3/executescript.py + + + .. method:: fetchone() + + Fetches the next row of a query result set, returning a single sequence, + or :const:`None` when no more data is available. + + + .. method:: fetchmany(size=cursor.arraysize) + + Fetches the next set of rows of a query result, returning a list. An empty + list is returned when no more rows are available. + + The number of rows to fetch per call is specified by the *size* parameter. + If it is not given, the cursor's arraysize determines the number of rows + to be fetched. The method should try to fetch as many rows as indicated by + the size parameter. If this is not possible due to the specified number of + rows not being available, fewer rows may be returned. + + Note there are performance considerations involved with the *size* parameter. + For optimal performance, it is usually best to use the arraysize attribute. + If the *size* parameter is used, then it is best for it to retain the same + value from one :meth:`fetchmany` call to the next. + + .. method:: fetchall() + + Fetches all (remaining) rows of a query result, returning a list. Note that + the cursor's arraysize attribute can affect the performance of this operation. + An empty list is returned when no rows are available. + + .. method:: close() + + Close the cursor now (rather than whenever ``__del__`` is called). + + The cursor will be unusable from this point forward; a :exc:`ProgrammingError` + exception will be raised if any operation is attempted with the cursor. + + .. attribute:: rowcount + + Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this + attribute, the database engine's own support for the determination of "rows + affected"/"rows selected" is quirky. + + For :meth:`executemany` statements, the number of modifications are summed up + into :attr:`rowcount`. + + As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in + case no ``executeXX()`` has been performed on the cursor or the rowcount of the + last operation is not determinable by the interface". This includes ``SELECT`` + statements because we cannot determine the number of rows a query produced + until all rows were fetched. + + .. attribute:: lastrowid + + This read-only attribute provides the rowid of the last modified row. It is + only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the + :meth:`execute` method. For operations other than ``INSERT`` or + ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is + set to :const:`None`. + + If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous + successful rowid is returned. + + .. versionchanged:: 3.6 + Added support for the ``REPLACE`` statement. + + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + + .. attribute:: description + + This read-only attribute provides the column names of the last query. To + remain compatible with the Python DB API, it returns a 7-tuple for each + column where the last six items of each tuple are :const:`None`. + + It is set for ``SELECT`` statements without any matching rows as well. + + .. attribute:: connection + + This read-only attribute provides the SQLite database :class:`Connection` + used by the :class:`Cursor` object. A :class:`Cursor` object created by + calling :meth:`con.cursor() ` will have a + :attr:`connection` attribute that refers to *con*:: + + >>> con = sqlite3.connect(":memory:") + >>> cur = con.cursor() + >>> cur.connection == con + True + +.. _sqlite3-row-objects: + +Row Objects +----------- + +.. class:: Row + + A :class:`Row` instance serves as a highly optimized + :attr:`~Connection.row_factory` for :class:`Connection` objects. + It tries to mimic a tuple in most of its features. + + It supports mapping access by column name and index, iteration, + representation, equality testing and :func:`len`. + + If two :class:`Row` objects have exactly the same columns and their + members are equal, they compare equal. + + .. method:: keys + + This method returns a list of column names. Immediately after a query, + it is the first member of each tuple in :attr:`Cursor.description`. + + .. versionchanged:: 3.5 + Added support of slicing. + +Let's assume we initialize a table as in the example given above:: + + con = sqlite3.connect(":memory:") + cur = con.cursor() + cur.execute('''create table stocks + (date text, trans text, symbol text, + qty real, price real)''') + cur.execute("""insert into stocks + values ('2006-01-05','BUY','RHAT',100,35.14)""") + con.commit() + cur.close() + +Now we plug :class:`Row` in:: + + >>> con.row_factory = sqlite3.Row + >>> cur = con.cursor() + >>> cur.execute('select * from stocks') + + >>> r = cur.fetchone() + >>> type(r) + + >>> tuple(r) + ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) + >>> len(r) + 5 + >>> r[2] + 'RHAT' + >>> r.keys() + ['date', 'trans', 'symbol', 'qty', 'price'] + >>> r['qty'] + 100.0 + >>> for member in r: + ... print(member) + ... + 2006-01-05 + BUY + RHAT + 100.0 + 35.14 + + +.. _sqlite3-exceptions: + +Exceptions +---------- + +.. exception:: Warning + + A subclass of :exc:`Exception`. + +.. exception:: Error + + The base class of the other exceptions in this module. It is a subclass + of :exc:`Exception`. + +.. exception:: DatabaseError + + Exception raised for errors that are related to the database. + +.. exception:: IntegrityError + + Exception raised when the relational integrity of the database is affected, + e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. + +.. exception:: ProgrammingError + + Exception raised for programming errors, e.g. table not found or already + exists, syntax error in the SQL statement, wrong number of parameters + specified, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: OperationalError + + Exception raised for errors that are related to the database's operation + and not necessarily under the control of the programmer, e.g. an unexpected + disconnect occurs, the data source name is not found, a transaction could + not be processed, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: NotSupportedError + + Exception raised in case a method or database API was used which is not + supported by the database, e.g. calling the :meth:`~Connection.rollback` + method on a connection that does not support transaction or has + transactions turned off. It is a subclass of :exc:`DatabaseError`. + + +.. _sqlite3-types: + +SQLite and Python types +----------------------- + + +Introduction +^^^^^^^^^^^^ + +SQLite natively supports the following types: ``NULL``, ``INTEGER``, +``REAL``, ``TEXT``, ``BLOB``. + +The following Python types can thus be sent to SQLite without any problem: + ++-------------------------------+-------------+ +| Python type | SQLite type | ++===============================+=============+ +| :const:`None` | ``NULL`` | ++-------------------------------+-------------+ +| :class:`int` | ``INTEGER`` | ++-------------------------------+-------------+ +| :class:`float` | ``REAL`` | ++-------------------------------+-------------+ +| :class:`str` | ``TEXT`` | ++-------------------------------+-------------+ +| :class:`bytes` | ``BLOB`` | ++-------------------------------+-------------+ + + +This is how SQLite types are converted to Python types by default: + ++-------------+----------------------------------------------+ +| SQLite type | Python type | ++=============+==============================================+ +| ``NULL`` | :const:`None` | ++-------------+----------------------------------------------+ +| ``INTEGER`` | :class:`int` | ++-------------+----------------------------------------------+ +| ``REAL`` | :class:`float` | ++-------------+----------------------------------------------+ +| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | +| | :class:`str` by default | ++-------------+----------------------------------------------+ +| ``BLOB`` | :class:`bytes` | ++-------------+----------------------------------------------+ + +The type system of the :mod:`sqlite3` module is extensible in two ways: you can +store additional Python types in a SQLite database via object adaptation, and +you can let the :mod:`sqlite3` module convert SQLite types to different Python +types via converters. + + +Using adapters to store additional Python types in SQLite databases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As described before, SQLite supports only a limited set of types natively. To +use other Python types with SQLite, you must **adapt** them to one of the +sqlite3 module's supported types for SQLite: one of NoneType, int, float, +str, bytes. + +There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python +type to one of the supported ones. + + +Letting your object adapt itself +"""""""""""""""""""""""""""""""" + +This is a good approach if you write the class yourself. Let's suppose you have +a class like this:: + + class Point: + def __init__(self, x, y): + self.x, self.y = x, y + +Now you want to store the point in a single SQLite column. First you'll have to +choose one of the supported types to be used for representing the point. +Let's just use str and separate the coordinates using a semicolon. Then you need +to give your class a method ``__conform__(self, protocol)`` which must return +the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. + +.. literalinclude:: ../includes/sqlite3/adapter_point_1.py + + +Registering an adapter callable +""""""""""""""""""""""""""""""" + +The other possibility is to create a function that converts the type to the +string representation and register the function with :meth:`register_adapter`. + +.. literalinclude:: ../includes/sqlite3/adapter_point_2.py + +The :mod:`sqlite3` module has two default adapters for Python's built-in +:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose +we want to store :class:`datetime.datetime` objects not in ISO representation, +but as a Unix timestamp. + +.. literalinclude:: ../includes/sqlite3/adapter_datetime.py + + +Converting SQLite values to custom Python types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Writing an adapter lets you send custom Python types to SQLite. But to make it +really useful we need to make the Python to SQLite to Python roundtrip work. + +Enter converters. + +Let's go back to the :class:`Point` class. We stored the x and y coordinates +separated via semicolons as strings in SQLite. + +First, we'll define a converter function that accepts the string as a parameter +and constructs a :class:`Point` object from it. + +.. note:: + + Converter functions **always** get called with a :class:`bytes` object, no + matter under which data type you sent the value to SQLite. + +:: + + def convert_point(s): + x, y = map(float, s.split(b";")) + return Point(x, y) + +Now you need to make the :mod:`sqlite3` module know that what you select from +the database is actually a point. There are two ways of doing this: + +* Implicitly via the declared type + +* Explicitly via the column name + +Both ways are described in section :ref:`sqlite3-module-contents`, in the entries +for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. + +The following example illustrates both approaches. + +.. literalinclude:: ../includes/sqlite3/converter_point.py + + +Default adapters and converters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are default adapters for the date and datetime types in the datetime +module. They will be sent as ISO dates/ISO timestamps to SQLite. + +The default converters are registered under the name "date" for +:class:`datetime.date` and under the name "timestamp" for +:class:`datetime.datetime`. + +This way, you can use date/timestamps from Python without any additional +fiddling in most cases. The format of the adapters is also compatible with the +experimental SQLite date/time functions. + +The following example demonstrates this. + +.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py + +If a timestamp stored in SQLite has a fractional part longer than 6 +numbers, its value will be truncated to microsecond precision by the +timestamp converter. + + +.. _sqlite3-controlling-transactions: + +Controlling Transactions +------------------------ + +The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, +but the Python :mod:`sqlite3` module by default does not. + +``autocommit`` mode means that statements that modify the database take effect +immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` +mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the +outermost transaction, turns ``autocommit`` mode back on. + +The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement +implicitly before a Data Modification Language (DML) statement (i.e. +``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). + +You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly +executes via the *isolation_level* parameter to the :func:`connect` +call, or via the :attr:`isolation_level` property of connections. +If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is +equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` +and ``EXCLUSIVE``. + +You can disable the :mod:`sqlite3` module's implicit transaction management by +setting :attr:`isolation_level` to ``None``. This will leave the underlying +``sqlite3`` library operating in ``autocommit`` mode. You can then completely +control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, +``SAVEPOINT``, and ``RELEASE`` statements in your code. + +.. versionchanged:: 3.6 + :mod:`sqlite3` used to implicitly commit an open transaction before DDL + statements. This is no longer the case. + + +Using :mod:`sqlite3` efficiently +-------------------------------- + + +Using shortcut methods +^^^^^^^^^^^^^^^^^^^^^^ + +Using the nonstandard :meth:`execute`, :meth:`executemany` and +:meth:`executescript` methods of the :class:`Connection` object, your code can +be written more concisely because you don't have to create the (often +superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` +objects are created implicitly and these shortcut methods return the cursor +objects. This way, you can execute a ``SELECT`` statement and iterate over it +directly using only a single call on the :class:`Connection` object. + +.. literalinclude:: ../includes/sqlite3/shortcut_methods.py + + +Accessing columns by name instead of by index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One useful feature of the :mod:`sqlite3` module is the built-in +:class:`sqlite3.Row` class designed to be used as a row factory. + +Rows wrapped with this class can be accessed both by index (like tuples) and +case-insensitively by name: + +.. literalinclude:: ../includes/sqlite3/rowclass.py + + +Using the connection as a context manager +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Connection objects can be used as context managers +that automatically commit or rollback transactions. In the event of an +exception, the transaction is rolled back; otherwise, the transaction is +committed: + +.. literalinclude:: ../includes/sqlite3/ctx_manager.py + + +.. rubric:: Footnotes + +.. [#f1] The sqlite3 module is not built with loadable extension support by + default, because some platforms (notably Mac OS X) have SQLite + libraries which are compiled without this feature. To get loadable + extension support, you must pass --enable-loadable-sqlite-extensions to + configure. diff --git a/Doc/library/statistics.rst.bak b/Doc/library/statistics.rst.bak new file mode 100644 index 000000000000000..6b6d3154a288100 --- /dev/null +++ b/Doc/library/statistics.rst.bak @@ -0,0 +1,879 @@ +:mod:`statistics` --- Mathematical statistics functions +======================================================= + +.. module:: statistics + :synopsis: Mathematical statistics functions + +.. moduleauthor:: Steven D'Aprano +.. sectionauthor:: Steven D'Aprano + +.. versionadded:: 3.4 + +**Source code:** :source:`Lib/statistics.py` + +.. testsetup:: * + + from statistics import * + __name__ = '' + +-------------- + +This module provides functions for calculating mathematical statistics of +numeric (:class:`~numbers.Real`-valued) data. + +The module is not intended to be a competitor to third-party libraries such +as `NumPy `_, `SciPy `_, or +proprietary full-featured statistics packages aimed at professional +statisticians such as Minitab, SAS and Matlab. It is aimed at the level of +graphing and scientific calculators. + +Unless explicitly noted, these functions support :class:`int`, +:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. +Behaviour with other types (whether in the numeric tower or not) is +currently unsupported. Collections with a mix of types are also undefined +and implementation-dependent. If your input data consists of mixed types, +you may be able to use :func:`map` to ensure a consistent result, for +example: ``map(float, input_data)``. + +Averages and measures of central location +----------------------------------------- + +These functions calculate an average or typical value from a population +or sample. + +======================= =============================================================== +:func:`mean` Arithmetic mean ("average") of data. +:func:`fmean` Fast, floating point arithmetic mean. +:func:`geometric_mean` Geometric mean of data. +:func:`harmonic_mean` Harmonic mean of data. +:func:`median` Median (middle value) of data. +:func:`median_low` Low median of data. +:func:`median_high` High median of data. +:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`mode` Single mode (most common value) of discrete or nominal data. +:func:`multimode` List of modes (most common values) of discrete or nomimal data. +:func:`quantiles` Divide data into intervals with equal probability. +======================= =============================================================== + +Measures of spread +------------------ + +These functions calculate a measure of how much the population or sample +tends to deviate from the typical or average values. + +======================= ============================================= +:func:`pstdev` Population standard deviation of data. +:func:`pvariance` Population variance of data. +:func:`stdev` Sample standard deviation of data. +:func:`variance` Sample variance of data. +======================= ============================================= + + +Function details +---------------- + +Note: The functions do not require the data given to them to be sorted. +However, for reading convenience, most of the examples show sorted sequences. + +.. function:: mean(data) + + Return the sample arithmetic mean of *data* which can be a sequence or iterable. + + The arithmetic mean is the sum of the data divided by the number of data + points. It is commonly called "the average", although it is only one of many + different mathematical averages. It is a measure of the central location of + the data. + + If *data* is empty, :exc:`StatisticsError` will be raised. + + Some examples of use: + + .. doctest:: + + >>> mean([1, 2, 3, 4, 4]) + 2.8 + >>> mean([-1.0, 2.5, 3.25, 5.75]) + 2.625 + + >>> from fractions import Fraction as F + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) + + >>> from decimal import Decimal as D + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + .. note:: + + The mean is strongly affected by outliers and is not a robust estimator + for central location: the mean is not necessarily a typical example of + the data points. For more robust measures of central location, see + :func:`median` and :func:`mode`. + + The sample mean gives an unbiased estimate of the true population mean, + so that when taken on average over all the possible samples, + ``mean(sample)`` converges on the true mean of the entire population. If + *data* represents the entire population rather than a sample, then + ``mean(data)`` is equivalent to calculating the true population mean μ. + + +.. function:: fmean(data) + + Convert *data* to floats and compute the arithmetic mean. + + This runs faster than the :func:`mean` function and it always returns a + :class:`float`. The *data* may be a sequence or iterable. If the input + dataset is empty, raises a :exc:`StatisticsError`. + + .. doctest:: + + >>> fmean([3.5, 4.0, 5.25]) + 4.25 + + .. versionadded:: 3.8 + + +.. function:: geometric_mean(data) + + Convert *data* to floats and compute the geometric mean. + + The geometric mean indicates the central tendency or typical value of the + *data* using the product of the values (as opposed to the arithmetic mean + which uses their sum). + + Raises a :exc:`StatisticsError` if the input dataset is empty, + if it contains a zero, or if it contains a negative value. + The *data* may be a sequence or iterable. + + No special efforts are made to achieve exact results. + (However, this may change in the future.) + + .. doctest:: + + >>> round(geometric_mean([54, 24, 36]), 1) + 36.0 + + .. versionadded:: 3.8 + + +.. function:: harmonic_mean(data, weights=None) + + Return the harmonic mean of *data*, a sequence or iterable of + real-valued numbers. If *weights* is omitted or *None*, then + equal weighting is assumed. + + The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the + reciprocals of the data. For example, the harmonic mean of three values *a*, + *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the + values is zero, the result will be zero. + + The harmonic mean is a type of average, a measure of the central + location of the data. It is often appropriate when averaging + ratios or rates, for example speeds. + + Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. + What is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60]) + 48.0 + + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 + + :exc:`StatisticsError` is raised if *data* is empty, any element + is less than zero, or if the weighted sum isn't positive. + + The current algorithm has an early-out when it encounters a zero + in the input. This means that the subsequent inputs are not tested + for validity. (This behavior may change in the future.) + + .. versionadded:: 3.6 + + .. versionchanged:: 3.10 + Added support for *weights*. + +.. function:: median(data) + + Return the median (middle value) of numeric data, using the common "mean of + middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. + *data* can be a sequence or iterable. + + The median is a robust measure of central location and is less affected by + the presence of outliers. When the number of data points is odd, the + middle data point is returned: + + .. doctest:: + + >>> median([1, 3, 5]) + 3 + + When the number of data points is even, the median is interpolated by taking + the average of the two middle values: + + .. doctest:: + + >>> median([1, 3, 5, 7]) + 4.0 + + This is suited for when your data is discrete, and you don't mind that the + median may not be an actual data point. + + If the data is ordinal (supports order operations) but not numeric (doesn't + support addition), consider using :func:`median_low` or :func:`median_high` + instead. + +.. function:: median_low(data) + + Return the low median of numeric data. If *data* is empty, + :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. + + The low median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the smaller of + the two middle values is returned. + + .. doctest:: + + >>> median_low([1, 3, 5]) + 3 + >>> median_low([1, 3, 5, 7]) + 3 + + Use the low median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_high(data) + + Return the high median of data. If *data* is empty, :exc:`StatisticsError` + is raised. *data* can be a sequence or iterable. + + The high median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the larger of + the two middle values is returned. + + .. doctest:: + + >>> median_high([1, 3, 5]) + 3 + >>> median_high([1, 3, 5, 7]) + 5 + + Use the high median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_grouped(data, interval=1) + + Return the median of grouped continuous data, calculated as the 50th + percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` + is raised. *data* can be a sequence or iterable. + + .. doctest:: + + >>> median_grouped([52, 52, 53, 54]) + 52.5 + + In the following example, the data are rounded, so that each value represents + the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 + is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data + given, the middle value falls somewhere in the class 3.5--4.5, and + interpolation is used to estimate it: + + .. doctest:: + + >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) + 3.7 + + Optional argument *interval* represents the class interval, and defaults + to 1. Changing the class interval naturally will change the interpolation: + + .. doctest:: + + >>> median_grouped([1, 3, 3, 5, 7], interval=1) + 3.25 + >>> median_grouped([1, 3, 3, 5, 7], interval=2) + 3.5 + + This function does not check whether the data points are at least + *interval* apart. + + .. impl-detail:: + + Under some circumstances, :func:`median_grouped` may coerce data points to + floats. This behaviour is likely to change in the future. + + .. seealso:: + + * "Statistics for the Behavioral Sciences", Frederick J Gravetter and + Larry B Wallnau (8th Edition). + + * The `SSMEDIAN + `_ + function in the Gnome Gnumeric spreadsheet, including `this discussion + `_. + + +.. function:: mode(data) + + Return the single most common data point from discrete or nominal *data*. + The mode (when it exists) is the most typical value and serves as a + measure of central location. + + If there are multiple modes with the same frequency, returns the first one + encountered in the *data*. If the smallest or largest of those is + desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. + If the input *data* is empty, :exc:`StatisticsError` is raised. + + ``mode`` assumes discrete data and returns a single value. This is the + standard treatment of the mode as commonly taught in schools: + + .. doctest:: + + >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) + 3 + + The mode is unique in that it is the only statistic in this package that + also applies to nominal (non-numeric) data: + + .. doctest:: + + >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) + 'red' + + .. versionchanged:: 3.8 + Now handles multimodal datasets by returning the first mode encountered. + Formerly, it raised :exc:`StatisticsError` when more than one mode was + found. + + +.. function:: multimode(data) + + Return a list of the most frequently occurring values in the order they + were first encountered in the *data*. Will return more than one result if + there are multiple modes or an empty list if the *data* is empty: + + .. doctest:: + + >>> multimode('aabbbbccddddeeffffgg') + ['b', 'd', 'f'] + >>> multimode('') + [] + + .. versionadded:: 3.8 + + +.. function:: pstdev(data, mu=None) + + Return the population standard deviation (the square root of the population + variance). See :func:`pvariance` for arguments and other details. + + .. doctest:: + + >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 0.986893273527251 + + +.. function:: pvariance(data, mu=None) + + Return the population variance of *data*, a non-empty sequence or iterable + of real-valued numbers. Variance, or second moment about the mean, is a + measure of the variability (spread or dispersion) of data. A large + variance indicates that the data is spread out; a small variance indicates + it is clustered closely around the mean. + + If the optional second argument *mu* is given, it is typically the mean of + the *data*. It can also be used to compute the second moment around a + point that is not the mean. If it is missing or ``None`` (the default), + the arithmetic mean is automatically calculated. + + Use this function to calculate the variance from the entire population. To + estimate the variance from a sample, the :func:`variance` function is usually + a better choice. + + Raises :exc:`StatisticsError` if *data* is empty. + + Examples: + + .. doctest:: + + >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] + >>> pvariance(data) + 1.25 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *mu* to avoid recalculation: + + .. doctest:: + + >>> mu = mean(data) + >>> pvariance(data, mu) + 1.25 + + Decimals and Fractions are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('24.815') + + >>> from fractions import Fraction as F + >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) + Fraction(13, 72) + + .. note:: + + When called with the entire population, this gives the population variance + σ². When called on a sample instead, this is the biased sample variance + s², also known as variance with N degrees of freedom. + + If you somehow know the true population mean μ, you may use this + function to calculate the variance of a sample, giving the known + population mean as the second argument. Provided the data points are a + random sample of the population, the result will be an unbiased estimate + of the population variance. + + +.. function:: stdev(data, xbar=None) + + Return the sample standard deviation (the square root of the sample + variance). See :func:`variance` for arguments and other details. + + .. doctest:: + + >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 1.0810874155219827 + + +.. function:: variance(data, xbar=None) + + Return the sample variance of *data*, an iterable of at least two real-valued + numbers. Variance, or second moment about the mean, is a measure of the + variability (spread or dispersion) of data. A large variance indicates that + the data is spread out; a small variance indicates it is clustered closely + around the mean. + + If the optional second argument *xbar* is given, it should be the mean of + *data*. If it is missing or ``None`` (the default), the mean is + automatically calculated. + + Use this function when your data is a sample from a population. To calculate + the variance from the entire population, see :func:`pvariance`. + + Raises :exc:`StatisticsError` if *data* has fewer than two values. + + Examples: + + .. doctest:: + + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *xbar* to avoid recalculation: + + .. doctest:: + + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 + + This function does not attempt to verify that you have passed the actual mean + as *xbar*. Using arbitrary values for *xbar* can lead to invalid or + impossible results. + + Decimal and Fraction values are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') + + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) + + .. note:: + + This is the sample variance s² with Bessel's correction, also known as + variance with N-1 degrees of freedom. Provided that the data points are + representative (e.g. independent and identically distributed), the result + should be an unbiased estimate of the true population variance. + + If you somehow know the actual population mean μ you should pass it to the + :func:`pvariance` function as the *mu* parameter to get the variance of a + sample. + +.. function:: quantiles(data, *, n=4, method='exclusive') + + Divide *data* into *n* continuous intervals with equal probability. + Returns a list of ``n - 1`` cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set + *n* to 100 for percentiles which gives the 99 cuts points that separate + *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* + is not least 1. + + The *data* can be any iterable containing sample data. For meaningful + results, the number of data points in *data* should be larger than *n*. + Raises :exc:`StatisticsError` if there are not at least two data points. + + The cut points are linearly interpolated from the + two nearest data points. For example, if a cut point falls one-third + of the distance between two sample values, ``100`` and ``112``, the + cut-point will evaluate to ``104``. + + The *method* for computing quantiles can be varied depending on + whether the *data* includes or excludes the lowest and + highest possible values from the population. + + The default *method* is "exclusive" and is used for data sampled from + a population that can have more extreme values than found in the + samples. The portion of the population falling below the *i-th* of + *m* sorted data points is computed as ``i / (m + 1)``. Given nine + sample values, the method sorts them and assigns the following + percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. + + Setting the *method* to "inclusive" is used for describing population + data or for samples that are known to include the most extreme values + from the population. The minimum value in *data* is treated as the 0th + percentile and the maximum value is treated as the 100th percentile. + The portion of the population falling below the *i-th* of *m* sorted + data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample + values, the method sorts them and assigns the following percentiles: + 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. + + .. doctest:: + + # Decile cut points for empirically sampled data + >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, + ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, + ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, + ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, + ... 103, 107, 101, 81, 109, 104] + >>> [round(q, 1) for q in quantiles(data, n=10)] + [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] + + .. versionadded:: 3.8 + + +Exceptions +---------- + +A single exception is defined: + +.. exception:: StatisticsError + + Subclass of :exc:`ValueError` for statistics-related exceptions. + + +:class:`NormalDist` objects +--------------------------- + +:class:`NormalDist` is a tool for creating and manipulating normal +distributions of a `random variable +`_. It is a +class that treats the mean and standard deviation of data +measurements as a single entity. + +Normal distributions arise from the `Central Limit Theorem +`_ and have a wide range +of applications in statistics. + +.. class:: NormalDist(mu=0.0, sigma=1.0) + + Returns a new *NormalDist* object where *mu* represents the `arithmetic + mean `_ and *sigma* + represents the `standard deviation + `_. + + If *sigma* is negative, raises :exc:`StatisticsError`. + + .. attribute:: mean + + A read-only property for the `arithmetic mean + `_ of a normal + distribution. + + .. attribute:: median + + A read-only property for the `median + `_ of a normal + distribution. + + .. attribute:: mode + + A read-only property for the `mode + `_ of a normal + distribution. + + .. attribute:: stdev + + A read-only property for the `standard deviation + `_ of a normal + distribution. + + .. attribute:: variance + + A read-only property for the `variance + `_ of a normal + distribution. Equal to the square of the standard deviation. + + .. classmethod:: NormalDist.from_samples(data) + + Makes a normal distribution instance with *mu* and *sigma* parameters + estimated from the *data* using :func:`fmean` and :func:`stdev`. + + The *data* can be any :term:`iterable` and should consist of values + that can be converted to type :class:`float`. If *data* does not + contain at least two elements, raises :exc:`StatisticsError` because it + takes at least one point to estimate a central value and at least two + points to estimate dispersion. + + .. method:: NormalDist.samples(n, *, seed=None) + + Generates *n* random samples for a given mean and standard deviation. + Returns a :class:`list` of :class:`float` values. + + If *seed* is given, creates a new instance of the underlying random + number generator. This is useful for creating reproducible results, + even in a multi-threading context. + + .. method:: NormalDist.pdf(x) + + Using a `probability density function (pdf) + `_, compute + the relative likelihood that a random variable *X* will be near the + given value *x*. Mathematically, it is the limit of the ratio ``P(x <= + X < x+dx) / dx`` as *dx* approaches zero. + + The relative likelihood is computed as the probability of a sample + occurring in a narrow range divided by the width of the range (hence + the word "density"). Since the likelihood is relative to other points, + its value can be greater than `1.0`. + + .. method:: NormalDist.cdf(x) + + Using a `cumulative distribution function (cdf) + `_, + compute the probability that a random variable *X* will be less than or + equal to *x*. Mathematically, it is written ``P(X <= x)``. + + .. method:: NormalDist.inv_cdf(p) + + Compute the inverse cumulative distribution function, also known as the + `quantile function `_ + or the `percent-point + `_ + function. Mathematically, it is written ``x : P(X <= x) = p``. + + Finds the value *x* of the random variable *X* such that the + probability of the variable being less than or equal to that value + equals the given probability *p*. + + .. method:: NormalDist.overlap(other) + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving `the overlapping area for + the two probability density functions + `_. + + .. method:: NormalDist.quantiles(n=4) + + Divide the normal distribution into *n* continuous intervals with + equal probability. Returns a list of (n - 1) cut points separating + the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution into 100 equal sized groups. + + .. method:: NormalDist.zscore(x) + + Compute the + `Standard Score `_ + describing *x* in terms of the number of standard deviations + above or below the mean of the normal distribution: + ``(x - mean) / stdev``. + + .. versionadded:: 3.9 + + Instances of :class:`NormalDist` support addition, subtraction, + multiplication and division by a constant. These operations + are used for translation and scaling. For example: + + .. doctest:: + + >>> temperature_february = NormalDist(5, 2.5) # Celsius + >>> temperature_february * (9/5) + 32 # Fahrenheit + NormalDist(mu=41.0, sigma=4.5) + + Dividing a constant by an instance of :class:`NormalDist` is not supported + because the result wouldn't be normally distributed. + + Since normal distributions arise from additive effects of independent + variables, it is possible to `add and subtract two independent normally + distributed random variables + `_ + represented as instances of :class:`NormalDist`. For example: + + .. doctest:: + + >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) + >>> drug_effects = NormalDist(0.4, 0.15) + >>> combined = birth_weights + drug_effects + >>> round(combined.mean, 1) + 3.1 + >>> round(combined.stdev, 1) + 0.5 + + .. versionadded:: 3.8 + + +:class:`NormalDist` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:class:`NormalDist` readily solves classic probability problems. + +For example, given `historical data for SAT exams +`_ showing +that scores are normally distributed with a mean of 1060 and a standard +deviation of 195, determine the percentage of students with test scores +between 1100 and 1200, after rounding to the nearest whole number: + +.. doctest:: + + >>> sat = NormalDist(1060, 195) + >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) + >>> round(fraction * 100.0, 1) + 18.4 + +Find the `quartiles `_ and `deciles +`_ for the SAT scores: + +.. doctest:: + + >>> list(map(round, sat.quantiles())) + [928, 1060, 1192] + >>> list(map(round, sat.quantiles(n=10))) + [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +To estimate the distribution for a model than isn't easy to solve +analytically, :class:`NormalDist` can generate input samples for a `Monte +Carlo simulation `_: + +.. doctest:: + + >>> def model(x, y, z): + ... return (3*x + 7*x*y - 5*y) / (11 * z) + ... + >>> n = 100_000 + >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) + >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) + >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) + >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP + [1.4591308524824727, 1.8035946855390597, 2.175091447274739] + +Normal distributions can be used to approximate `Binomial +distributions `_ +when the sample size is large and when the probability of a successful +trial is near 50%. + +For example, an open source conference has 750 attendees and two rooms with a +500 person capacity. There is a talk about Python and another about Ruby. +In previous conferences, 65% of the attendees preferred to listen to Python +talks. Assuming the population preferences haven't changed, what is the +probability that the Python room will stay within its capacity limits? + +.. doctest:: + + >>> n = 750 # Sample size + >>> p = 0.65 # Preference for Python + >>> q = 1.0 - p # Preference for Ruby + >>> k = 500 # Room capacity + + >>> # Approximation using the cumulative normal distribution + >>> from math import sqrt + >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) + 0.8402 + + >>> # Solution using the cumulative binomial distribution + >>> from math import comb, fsum + >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) + 0.8402 + + >>> # Approximation using a simulation + >>> from random import seed, choices + >>> seed(8675309) + >>> def trial(): + ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') + >>> mean(trial() <= k for i in range(10_000)) + 0.8398 + +Normal distributions commonly arise in machine learning problems. + +Wikipedia has a `nice example of a Naive Bayesian Classifier +`_. +The challenge is to predict a person's gender from measurements of normally +distributed features including height, weight, and foot size. + +We're given a training dataset with measurements for eight people. The +measurements are assumed to be normally distributed, so we summarize the data +with :class:`NormalDist`: + +.. doctest:: + + >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) + >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) + >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) + >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) + >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) + >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) + +Next, we encounter a new person whose feature measurements are known but whose +gender is unknown: + +.. doctest:: + + >>> ht = 6.0 # height + >>> wt = 130 # weight + >>> fs = 8 # foot size + +Starting with a 50% `prior probability +`_ of being male or female, +we compute the posterior as the prior times the product of likelihoods for the +feature measurements given the gender: + +.. doctest:: + + >>> prior_male = 0.5 + >>> prior_female = 0.5 + >>> posterior_male = (prior_male * height_male.pdf(ht) * + ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) + + >>> posterior_female = (prior_female * height_female.pdf(ht) * + ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) + +The final prediction goes to the largest posterior. This is known as the +`maximum a posteriori +`_ or MAP: + +.. doctest:: + + >>> 'male' if posterior_male > posterior_female else 'female' + 'female' + + +.. + # This modelines must appear within the last ten lines of the file. + kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/tempfile.rst.bak b/Doc/library/tempfile.rst.bak new file mode 100644 index 000000000000000..2b8a35e2651e731 --- /dev/null +++ b/Doc/library/tempfile.rst.bak @@ -0,0 +1,374 @@ +:mod:`tempfile` --- Generate temporary files and directories +============================================================ + +.. module:: tempfile + :synopsis: Generate temporary files and directories. + +.. sectionauthor:: Zack Weinberg + +**Source code:** :source:`Lib/tempfile.py` + +.. index:: + pair: temporary; file name + pair: temporary; file + +-------------- + +This module creates temporary files and directories. It works on all +supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, +:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level +interfaces which provide automatic cleanup and can be used as +context managers. :func:`mkstemp` and +:func:`mkdtemp` are lower-level functions which require manual cleanup. + +All the user-callable functions and constructors take additional arguments which +allow direct control over the location and name of temporary files and +directories. Files names used by this module include a string of +random characters which allows those files to be securely created in +shared temporary directories. +To maintain backward compatibility, the argument order is somewhat odd; it +is recommended to use keyword arguments for clarity. + +The module defines the following user-callable items: + +.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) + + Return a :term:`file-like object` that can be used as a temporary storage area. + The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon + as it is closed (including an implicit close when the object is garbage + collected). Under Unix, the directory entry for the file is either not created at all or is removed + immediately after the file is created. Other platforms do not support + this; your code should not rely on a temporary file created using this + function having or not having a visible name in the file system. + + The resulting object can be used as a context manager (see + :ref:`tempfile-examples`). On completion of the context or + destruction of the file object the temporary file will be removed + from the filesystem. + + The *mode* parameter defaults to ``'w+b'`` so that the file created can + be read and written without being closed. Binary mode is used so that it + behaves consistently on all platforms without regard for the data that is + stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for + :func:`open`. + + The *dir*, *prefix* and *suffix* parameters have the same meaning and + defaults as with :func:`mkstemp`. + + The returned object is a true file object on POSIX platforms. On other + platforms, it is a file-like object whose :attr:`!file` attribute is the + underlying true file object. + + The :py:data:`os.O_TMPFILE` flag is used if it is available and works + (Linux-specific, requires Linux kernel 3.11 or later). + + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile + + .. versionchanged:: 3.5 + + The :py:data:`os.O_TMPFILE` flag is now used if available. + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) + + This function operates exactly as :func:`TemporaryFile` does, except that + the file is guaranteed to have a visible name in the file system (on + Unix, the directory entry is not unlinked). That name can be retrieved + from the :attr:`name` attribute of the returned + file-like object. Whether the name can be + used to open the file a second time, while the named temporary file is + still open, varies across platforms (it can be so used on Unix; it cannot + on Windows NT or later). If *delete* is true (the default), the file is + deleted as soon as it is closed. + The returned object is always a file-like object whose :attr:`!file` + attribute is the underlying true file object. This file-like object can + be used in a :keyword:`with` statement, just like a normal file. + + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) + + This function operates exactly as :func:`TemporaryFile` does, except that + data is spooled in memory until the file size exceeds *max_size*, or + until the file's :func:`fileno` method is called, at which point the + contents are written to disk and operation proceeds as with + :func:`TemporaryFile`. + + The resulting file has one additional method, :func:`rollover`, which + causes the file to roll over to an on-disk file regardless of its size. + + The returned object is a file-like object whose :attr:`_file` attribute + is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object + (depending on whether binary or text *mode* was specified) or a true file + object, depending on whether :func:`rollover` has been called. This + file-like object can be used in a :keyword:`with` statement, just like + a normal file. + + .. versionchanged:: 3.3 + the truncate method now accepts a ``size`` argument. + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) + + This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. + The resulting object can be used as a context manager (see + :ref:`tempfile-examples`). On completion of the context or destruction + of the temporary directory object the newly created temporary directory + and all its contents are removed from the filesystem. + + The directory name can be retrieved from the :attr:`name` attribute of the + returned object. When the returned object is used as a context manager, the + :attr:`name` will be assigned to the target of the :keyword:`!as` clause in + the :keyword:`with` statement, if there is one. + + The directory can be explicitly cleaned up by calling the + :func:`cleanup` method. + + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory + + .. versionadded:: 3.2 + + +.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) + + Creates a temporary file in the most secure manner possible. There are + no race conditions in the file's creation, assuming that the platform + properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The + file is readable and writable only by the creating user ID. If the + platform uses permission bits to indicate whether a file is executable, + the file is executable by no one. The file descriptor is not inherited + by child processes. + + Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible + for deleting the temporary file when done with it. + + If *suffix* is not ``None``, the file name will end with that suffix, + otherwise there will be no suffix. :func:`mkstemp` does not put a dot + between the file name and the suffix; if you need one, put it at the + beginning of *suffix*. + + If *prefix* is not ``None``, the file name will begin with that prefix; + otherwise, a default prefix is used. The default is the return value of + :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. + + If *dir* is not ``None``, the file will be created in that directory; + otherwise, a default directory is used. The default directory is chosen + from a platform-dependent list, but the user of the application can + control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* + environment variables. There is thus no guarantee that the generated + filename will have any nice properties, such as not requiring quoting + when passed to external commands via ``os.popen()``. + + If any of *suffix*, *prefix*, and *dir* are not + ``None``, they must be the same type. + If they are bytes, the returned name will be bytes instead of str. + If you want to force a bytes return value with otherwise default behavior, + pass ``suffix=b''``. + + If *text* is specified and true, the file is opened in text mode. + Otherwise, (the default) the file is opened in binary mode. + + :func:`mkstemp` returns a tuple containing an OS-level handle to an open + file (as would be returned by :func:`os.open`) and the absolute pathname + of that file, in that order. + + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp + + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + + +.. function:: mkdtemp(suffix=None, prefix=None, dir=None) + + Creates a temporary directory in the most secure manner possible. There + are no race conditions in the directory's creation. The directory is + readable, writable, and searchable only by the creating user ID. + + The user of :func:`mkdtemp` is responsible for deleting the temporary + directory and its contents when done with it. + + The *prefix*, *suffix*, and *dir* arguments are the same as for + :func:`mkstemp`. + + :func:`mkdtemp` returns the absolute pathname of the new directory. + + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp + + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + + +.. function:: gettempdir() + + Return the name of the directory used for temporary files. This + defines the default value for the *dir* argument to all functions + in this module. + + Python searches a standard list of directories to find one which + the calling user can create files in. The list is: + + #. The directory named by the :envvar:`TMPDIR` environment variable. + + #. The directory named by the :envvar:`TEMP` environment variable. + + #. The directory named by the :envvar:`TMP` environment variable. + + #. A platform-specific location: + + * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, + :file:`\\TEMP`, and :file:`\\TMP`, in that order. + + * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and + :file:`/usr/tmp`, in that order. + + #. As a last resort, the current working directory. + + The result of this search is cached, see the description of + :data:`tempdir` below. + + .. versionchanged:: 3.10 + + Always returns a str. Previously it would return any :data:`tempdir` + value regardless of type so long as it was not ``None``. + +.. function:: gettempdirb() + + Same as :func:`gettempdir` but the return value is in bytes. + + .. versionadded:: 3.5 + +.. function:: gettempprefix() + + Return the filename prefix used to create temporary files. This does not + contain the directory component. + +.. function:: gettempprefixb() + + Same as :func:`gettempprefix` but the return value is in bytes. + + .. versionadded:: 3.5 + +The module uses a global variable to store the name of the directory +used for temporary files returned by :func:`gettempdir`. It can be +set directly to override the selection process, but this is discouraged. +All functions in this module take a *dir* argument which can be used +to specify the directory. This is the recommended approach that does +not surprise other unsuspecting code by changing global API behavior. + +.. data:: tempdir + + When set to a value other than ``None``, this variable defines the + default value for the *dir* argument to the functions defined in this + module, including its type, bytes or str. It cannot be a + :term:`path-like object`. + + If ``tempdir`` is ``None`` (the default) at any call to any of the above + functions except :func:`gettempprefix` it is initialized following the + algorithm described in :func:`gettempdir`. + + .. note:: + + Beware that if you set ``tempdir`` to a bytes value, there is a + nasty side effect: The global default return type of + :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no + explicit ``prefix``, ``suffix``, or ``dir`` arguments of type + str are supplied. Please do not write code expecting or + depending on this. This awkward behavior is maintained for + compatibility with the historcal implementation. + +.. _tempfile-examples: + +Examples +-------- + +Here are some examples of typical usage of the :mod:`tempfile` module:: + + >>> import tempfile + + # create a temporary file and write some data to it + >>> fp = tempfile.TemporaryFile() + >>> fp.write(b'Hello world!') + # read data from file + >>> fp.seek(0) + >>> fp.read() + b'Hello world!' + # close the file, it will be removed + >>> fp.close() + + # create a temporary file using a context manager + >>> with tempfile.TemporaryFile() as fp: + ... fp.write(b'Hello world!') + ... fp.seek(0) + ... fp.read() + b'Hello world!' + >>> + # file is now closed and removed + + # create a temporary directory using the context manager + >>> with tempfile.TemporaryDirectory() as tmpdirname: + ... print('created temporary directory', tmpdirname) + >>> + # directory and contents have been removed + + +Deprecated functions and variables +---------------------------------- + +A historical way to create temporary files was to first generate a +file name with the :func:`mktemp` function and then create a file +using this name. Unfortunately this is not secure, because a different +process may create a file with this name in the time between the call +to :func:`mktemp` and the subsequent attempt to create the file by the +first process. The solution is to combine the two steps and create the +file immediately. This approach is used by :func:`mkstemp` and the +other functions described above. + +.. function:: mktemp(suffix='', prefix='tmp', dir=None) + + .. deprecated:: 2.3 + Use :func:`mkstemp` instead. + + Return an absolute pathname of a file that did not exist at the time the + call is made. The *prefix*, *suffix*, and *dir* arguments are similar + to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` + and ``prefix=None`` are not supported. + + .. warning:: + + Use of this function may introduce a security hole in your program. By + the time you get around to doing anything with the file name it returns, + someone else may have beaten you to the punch. :func:`mktemp` usage can + be replaced easily with :func:`NamedTemporaryFile`, passing it the + ``delete=False`` parameter:: + + >>> f = NamedTemporaryFile(delete=False) + >>> f.name + '/tmp/tmptjujjt' + >>> f.write(b"Hello World!\n") + 13 + >>> f.close() + >>> os.unlink(f.name) + >>> os.path.exists(f.name) + False diff --git a/Doc/library/venv.rst.bak b/Doc/library/venv.rst.bak new file mode 100644 index 000000000000000..5d4a36481f1dcc5 --- /dev/null +++ b/Doc/library/venv.rst.bak @@ -0,0 +1,496 @@ +:mod:`venv` --- Creation of virtual environments +================================================ + +.. module:: venv + :synopsis: Creation of virtual environments. + +.. moduleauthor:: Vinay Sajip +.. sectionauthor:: Vinay Sajip + +.. versionadded:: 3.3 + +**Source code:** :source:`Lib/venv/` + +.. index:: pair: Environments; virtual + +-------------- + +The :mod:`venv` module provides support for creating lightweight "virtual +environments" with their own site directories, optionally isolated from system +site directories. Each virtual environment has its own Python binary (which +matches the version of the binary that was used to create this environment) and +can have its own independent set of installed Python packages in its site +directories. + +See :pep:`405` for more information about Python virtual environments. + +.. seealso:: + + `Python Packaging User Guide: Creating and using virtual environments + `__ + + +Creating virtual environments +----------------------------- + +.. include:: /using/venv-create.inc + + +.. _venv-def: + +.. note:: A virtual environment is a Python environment such that the Python + interpreter, libraries and scripts installed into it are isolated from those + installed in other virtual environments, and (by default) any libraries + installed in a "system" Python, i.e., one which is installed as part of your + operating system. + + A virtual environment is a directory tree which contains Python executable + files and other files which indicate that it is a virtual environment. + + Common installation tools such as setuptools_ and pip_ work as + expected with virtual environments. In other words, when a virtual + environment is active, they install Python packages into the virtual + environment without needing to be told to do so explicitly. + + When a virtual environment is active (i.e., the virtual environment's Python + interpreter is running), the attributes :attr:`sys.prefix` and + :attr:`sys.exec_prefix` point to the base directory of the virtual + environment, whereas :attr:`sys.base_prefix` and + :attr:`sys.base_exec_prefix` point to the non-virtual environment Python + installation which was used to create the virtual environment. If a virtual + environment is not active, then :attr:`sys.prefix` is the same as + :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as + :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment + Python installation). + + When a virtual environment is active, any options that change the + installation path will be ignored from all :mod:`distutils` configuration + files to prevent projects being inadvertently installed outside of the + virtual environment. + + When working in a command shell, users can make a virtual environment active + by running an ``activate`` script in the virtual environment's executables + directory (the precise filename and command to use the file is + shell-dependent), which prepends the virtual environment's directory for + executables to the ``PATH`` environment variable for the running shell. There + should be no need in other circumstances to activate a virtual + environment; scripts installed into virtual environments have a "shebang" + line which points to the virtual environment's Python interpreter. This means + that the script will run with that interpreter regardless of the value of + ``PATH``. On Windows, "shebang" line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in a + Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its virtual environment in + ``PATH``. + + +.. _venv-api: + +API +--- + +.. highlight:: python + +The high-level method described above makes use of a simple API which provides +mechanisms for third-party virtual environment creators to customize environment +creation according to their needs, the :class:`EnvBuilder` class. + +.. class:: EnvBuilder(system_site_packages=False, clear=False, \ + symlinks=False, upgrade=False, with_pip=False, \ + prompt=None, upgrade_deps=False) + + The :class:`EnvBuilder` class accepts the following keyword arguments on + instantiation: + + * ``system_site_packages`` -- a Boolean value indicating that the system Python + site-packages should be available to the environment (defaults to ``False``). + + * ``clear`` -- a Boolean value which, if true, will delete the contents of + any existing target directory, before creating the environment. + + * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + Python binary rather than copying. + + * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + environment with the running Python - for use when that Python has been + upgraded in-place (defaults to ``False``). + + * ``with_pip`` -- a Boolean value which, if true, ensures pip is + installed in the virtual environment. This uses :mod:`ensurepip` with + the ``--default-pip`` option. + + * ``prompt`` -- a String to be used after virtual environment is activated + (defaults to ``None`` which means directory name of the environment would + be used). If the special string ``"."`` is provided, the basename of the + current directory is used as the prompt. + + * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + + .. versionadded:: 3.6 + Added the ``prompt`` parameter + + .. versionadded:: 3.9 + Added the ``upgrade_deps`` parameter + + Creators of third-party virtual environment tools will be free to use the + provided :class:`EnvBuilder` class as a base class. + + The returned env-builder is an object which has a method, ``create``: + + .. method:: create(env_dir) + + Create a virtual environment by specifying the target directory + (absolute or relative to the current directory) which is to contain the + virtual environment. The ``create`` method will either create the + environment in the specified directory, or raise an appropriate + exception. + + The ``create`` method of the :class:`EnvBuilder` class illustrates the + hooks available for subclass customization:: + + def create(self, env_dir): + """ + Create a virtualized Python environment in a directory. + env_dir is the target directory to create an environment in. + """ + env_dir = os.path.abspath(env_dir) + context = self.ensure_directories(env_dir) + self.create_configuration(context) + self.setup_python(context) + self.setup_scripts(context) + self.post_setup(context) + + Each of the methods :meth:`ensure_directories`, + :meth:`create_configuration`, :meth:`setup_python`, + :meth:`setup_scripts` and :meth:`post_setup` can be overridden. + + .. method:: ensure_directories(env_dir) + + Creates the environment directory and all necessary directories, and + returns a context object. This is just a holder for attributes (such as + paths), for use by the other methods. The directories are allowed to + exist already, as long as either ``clear`` or ``upgrade`` were + specified to allow operating on an existing environment directory. + + .. method:: create_configuration(context) + + Creates the ``pyvenv.cfg`` configuration file in the environment. + + .. method:: setup_python(context) + + Creates a copy or symlink to the Python executable in the environment. + On POSIX systems, if a specific executable ``python3.x`` was used, + symlinks to ``python`` and ``python3`` will be created pointing to that + executable, unless files with those names already exist. + + .. method:: setup_scripts(context) + + Installs activation scripts appropriate to the platform into the virtual + environment. + + .. method:: upgrade_dependencies(context) + + Upgrades the core venv dependency packages (currently ``pip`` and + ``setuptools``) in the environment. This is done by shelling out to the + ``pip`` executable in the environment. + + .. versionadded:: 3.9 + + .. method:: post_setup(context) + + A placeholder method which can be overridden in third party + implementations to pre-install packages in the virtual environment or + perform other post-creation steps. + + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + + In addition, :class:`EnvBuilder` provides this utility method that can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + + .. method:: install_scripts(context, path) + + *path* is the path to a directory that should contain subdirectories + "common", "posix", "nt", each containing scripts destined for the bin + directory in the environment. The contents of "common" and the + directory corresponding to :data:`os.name` are copied after some text + replacement of placeholders: + + * ``__VENV_DIR__`` is replaced with the absolute path of the environment + directory. + + * ``__VENV_NAME__`` is replaced with the environment name (final path + segment of environment directory). + + * ``__VENV_PROMPT__`` is replaced with the prompt (the environment + name surrounded by parentheses and with a following space) + + * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory + (either ``bin`` or ``Scripts``). + + * ``__VENV_PYTHON__`` is replaced with the absolute path of the + environment's executable. + + The directories are allowed to exist (for when an existing environment + is being upgraded). + +There is also a module-level convenience function: + +.. function:: create(env_dir, system_site_packages=False, clear=False, \ + symlinks=False, with_pip=False, prompt=None, \ + upgrade_deps=False) + + Create an :class:`EnvBuilder` with the given keyword arguments, and call its + :meth:`~EnvBuilder.create` method with the *env_dir* argument. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + + .. versionchanged:: 3.6 + Added the ``prompt`` parameter + + .. versionchanged:: 3.9 + Added the ``upgrade_deps`` parameter + +An example of extending ``EnvBuilder`` +-------------------------------------- + +The following script shows how to extend :class:`EnvBuilder` by implementing a +subclass which installs setuptools and pip into a created virtual environment:: + + import os + import os.path + from subprocess import Popen, PIPE + import sys + from threading import Thread + from urllib.parse import urlparse + from urllib.request import urlretrieve + import venv + + class ExtendedEnvBuilder(venv.EnvBuilder): + """ + This builder installs setuptools and pip so that you can pip or + easy_install other packages into the created virtual environment. + + :param nodist: If true, setuptools and pip are not installed into the + created virtual environment. + :param nopip: If true, pip is not installed into the created + virtual environment. + :param progress: If setuptools or pip are installed, the progress of the + installation can be monitored by passing a progress + callable. If specified, it is called with two + arguments: a string indicating some progress, and a + context indicating where the string is coming from. + The context argument can have one of three values: + 'main', indicating that it is called from virtualize() + itself, and 'stdout' and 'stderr', which are obtained + by reading lines from the output streams of a subprocess + which is used to install the app. + + If a callable is not specified, default progress + information is output to sys.stderr. + """ + + def __init__(self, *args, **kwargs): + self.nodist = kwargs.pop('nodist', False) + self.nopip = kwargs.pop('nopip', False) + self.progress = kwargs.pop('progress', None) + self.verbose = kwargs.pop('verbose', False) + super().__init__(*args, **kwargs) + + def post_setup(self, context): + """ + Set up any packages which need to be pre-installed into the + virtual environment being created. + + :param context: The information for the virtual environment + creation request being processed. + """ + os.environ['VIRTUAL_ENV'] = context.env_dir + if not self.nodist: + self.install_setuptools(context) + # Can't install pip without setuptools + if not self.nopip and not self.nodist: + self.install_pip(context) + + def reader(self, stream, context): + """ + Read lines from a subprocess' output stream and either pass to a progress + callable (if specified) or write progress information to sys.stderr. + """ + progress = self.progress + while True: + s = stream.readline() + if not s: + break + if progress is not None: + progress(s, context) + else: + if not self.verbose: + sys.stderr.write('.') + else: + sys.stderr.write(s.decode('utf-8')) + sys.stderr.flush() + stream.close() + + def install_script(self, context, name, url): + _, _, path, _, _, _ = urlparse(url) + fn = os.path.split(path)[-1] + binpath = context.bin_path + distpath = os.path.join(binpath, fn) + # Download script into the virtual environment's binaries folder + urlretrieve(url, distpath) + progress = self.progress + if self.verbose: + term = '\n' + else: + term = '' + if progress is not None: + progress('Installing %s ...%s' % (name, term), 'main') + else: + sys.stderr.write('Installing %s ...%s' % (name, term)) + sys.stderr.flush() + # Install in the virtual environment + args = [context.env_exe, fn] + p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) + t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) + t1.start() + t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) + t2.start() + p.wait() + t1.join() + t2.join() + if progress is not None: + progress('done.', 'main') + else: + sys.stderr.write('done.\n') + # Clean up - no longer needed + os.unlink(distpath) + + def install_setuptools(self, context): + """ + Install setuptools in the virtual environment. + + :param context: The information for the virtual environment + creation request being processed. + """ + url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' + self.install_script(context, 'setuptools', url) + # clear up the setuptools archive which gets downloaded + pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') + files = filter(pred, os.listdir(context.bin_path)) + for f in files: + f = os.path.join(context.bin_path, f) + os.unlink(f) + + def install_pip(self, context): + """ + Install pip in the virtual environment. + + :param context: The information for the virtual environment + creation request being processed. + """ + url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' + self.install_script(context, 'pip', url) + + def main(args=None): + compatible = True + if sys.version_info < (3, 3): + compatible = False + elif not hasattr(sys, 'base_prefix'): + compatible = False + if not compatible: + raise ValueError('This script is only for use with ' + 'Python 3.3 or later') + else: + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory in which to create the + 'virtual environment.') + parser.add_argument('--no-setuptools', default=False, + action='store_true', dest='nodist', + help="Don't install setuptools or pip in the " + "virtual environment.") + parser.add_argument('--no-pip', default=False, + action='store_true', dest='nopip', + help="Don't install pip in the virtual " + "environment.") + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False + else: + use_symlinks = True + parser.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'virtual environment ' + 'directory if it already ' + 'exists, before virtual ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the virtual ' + 'environment directory to ' + 'use this version of ' + 'Python, assuming Python ' + 'has been upgraded ' + 'in-place.') + parser.add_argument('--verbose', default=False, action='store_true', + dest='verbose', help='Display the output ' + 'from the scripts which ' + 'install setuptools and pip.') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = ExtendedEnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + nodist=options.nodist, + nopip=options.nopip, + verbose=options.verbose) + for d in options.dirs: + builder.create(d) + + if __name__ == '__main__': + rc = 1 + try: + main() + rc = 0 + except Exception as e: + print('Error: %s' % e, file=sys.stderr) + sys.exit(rc) + + +This script is also available for download `online +`_. + + +.. _setuptools: https://pypi.org/project/setuptools/ +.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/xml.sax.handler.rst.bak b/Doc/library/xml.sax.handler.rst.bak new file mode 100644 index 000000000000000..3746a58c9b9558a --- /dev/null +++ b/Doc/library/xml.sax.handler.rst.bak @@ -0,0 +1,463 @@ +:mod:`xml.sax.handler` --- Base classes for SAX handlers +======================================================== + +.. module:: xml.sax.handler + :synopsis: Base classes for SAX event handlers. + +.. moduleauthor:: Lars Marius Garshol +.. sectionauthor:: Martin v. Löwis + +**Source code:** :source:`Lib/xml/sax/handler.py` + +-------------- + +The SAX API defines five kinds of handlers: content handlers, DTD handlers, +error handlers, entity resolvers and lexical handlers. Applications normally +only need to implement those interfaces whose events they are interested in; +they can implement the interfaces in a single object or in multiple objects. +Handler implementations should inherit from the base classes provided in the +module :mod:`xml.sax.handler`, so that all methods get default implementations. + + +.. class:: ContentHandler + + This is the main callback interface in SAX, and the one most important to + applications. The order of events in this interface mirrors the order of the + information in the document. + + +.. class:: DTDHandler + + Handle DTD events. + + This interface specifies only those DTD events required for basic parsing + (unparsed entities and attributes). + + +.. class:: EntityResolver + + Basic interface for resolving entities. If you create an object implementing + this interface, then register the object with your Parser, the parser will call + the method in your object to resolve all external entities. + + +.. class:: ErrorHandler + + Interface used by the parser to present error and warning messages to the + application. The methods of this object control whether errors are immediately + converted to exceptions or are handled in some other way. + + +.. class:: LexicalHandler + + Interface used by the parser to represent low freqency events which may not + be of interest to many applications. + +In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants +for the feature and property names. + + +.. data:: feature_namespaces + + | value: ``"http://xml.org/sax/features/namespaces"`` + | true: Perform Namespace processing. + | false: Optionally do not perform Namespace processing (implies + namespace-prefixes; default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_namespace_prefixes + + | value: ``"http://xml.org/sax/features/namespace-prefixes"`` + | true: Report the original prefixed names and attributes used for Namespace + declarations. + | false: Do not report attributes used for Namespace declarations, and + optionally do not report original prefixed names (default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_string_interning + + | value: ``"http://xml.org/sax/features/string-interning"`` + | true: All element names, prefixes, attribute names, Namespace URIs, and + local names are interned using the built-in intern function. + | false: Names are not necessarily interned, although they may be (default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_validation + + | value: ``"http://xml.org/sax/features/validation"`` + | true: Report all validation errors (implies external-general-entities and + external-parameter-entities). + | false: Do not report validation errors. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_external_ges + + | value: ``"http://xml.org/sax/features/external-general-entities"`` + | true: Include all external general (text) entities. + | false: Do not include external general entities. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_external_pes + + | value: ``"http://xml.org/sax/features/external-parameter-entities"`` + | true: Include all external parameter entities, including the external DTD + subset. + | false: Do not include any external parameter entities, even the external + DTD subset. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: all_features + + List of all features. + + +.. data:: property_lexical_handler + + | value: ``"http://xml.org/sax/properties/lexical-handler"`` + | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) + | description: An optional extension handler for lexical events like + comments. + | access: read/write + + +.. data:: property_declaration_handler + + | value: ``"http://xml.org/sax/properties/declaration-handler"`` + | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) + | description: An optional extension handler for DTD-related events other + than notations and unparsed entities. + | access: read/write + + +.. data:: property_dom_node + + | value: ``"http://xml.org/sax/properties/dom-node"`` + | data type: org.w3c.dom.Node (not supported in Python 2) + | description: When parsing, the current DOM node being visited if this is + a DOM iterator; when not parsing, the root DOM node for iteration. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: property_xml_string + + | value: ``"http://xml.org/sax/properties/xml-string"`` + | data type: String + | description: The literal string of characters that was the source for the + current event. + | access: read-only + + +.. data:: all_properties + + List of all known property names. + + +.. _content-handler-objects: + +ContentHandler Objects +---------------------- + +Users are expected to subclass :class:`ContentHandler` to support their +application. The following methods are called by the parser on the appropriate +events in the input document: + + +.. method:: ContentHandler.setDocumentLocator(locator) + + Called by the parser to give the application a locator for locating the origin + of document events. + + SAX parsers are strongly encouraged (though not absolutely required) to supply a + locator: if it does so, it must supply the locator to the application by + invoking this method before invoking any of the other methods in the + DocumentHandler interface. + + The locator allows the application to determine the end position of any + document-related event, even if the parser is not reporting an error. Typically, + the application will use this information for reporting its own errors (such as + character content that does not match an application's business rules). The + information returned by the locator is probably not sufficient for use with a + search engine. + + Note that the locator will return correct information only during the invocation + of the events in this interface. The application should not attempt to use it at + any other time. + + +.. method:: ContentHandler.startDocument() + + Receive notification of the beginning of a document. + + The SAX parser will invoke this method only once, before any other methods in + this interface or in DTDHandler (except for :meth:`setDocumentLocator`). + + +.. method:: ContentHandler.endDocument() + + Receive notification of the end of a document. + + The SAX parser will invoke this method only once, and it will be the last method + invoked during the parse. The parser shall not invoke this method until it has + either abandoned parsing (because of an unrecoverable error) or reached the end + of input. + + +.. method:: ContentHandler.startPrefixMapping(prefix, uri) + + Begin the scope of a prefix-URI Namespace mapping. + + The information from this event is not necessary for normal Namespace + processing: the SAX XML reader will automatically replace prefixes for element + and attribute names when the ``feature_namespaces`` feature is enabled (the + default). + + There are cases, however, when applications need to use prefixes in character + data or in attribute values, where they cannot safely be expanded automatically; + the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the + information to the application to expand prefixes in those contexts itself, if + necessary. + + .. XXX This is not really the default, is it? MvL + + Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not + guaranteed to be properly nested relative to each-other: all + :meth:`startPrefixMapping` events will occur before the corresponding + :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur + after the corresponding :meth:`endElement` event, but their order is not + guaranteed. + + +.. method:: ContentHandler.endPrefixMapping(prefix) + + End the scope of a prefix-URI mapping. + + See :meth:`startPrefixMapping` for details. This event will always occur after + the corresponding :meth:`endElement` event, but the order of + :meth:`endPrefixMapping` events is not otherwise guaranteed. + + +.. method:: ContentHandler.startElement(name, attrs) + + Signals the start of an element in non-namespace mode. + + The *name* parameter contains the raw XML 1.0 name of the element type as a + string and the *attrs* parameter holds an object of the + :class:`~xml.sax.xmlreader.Attributes` + interface (see :ref:`attributes-objects`) containing the attributes of + the element. The object passed as *attrs* may be re-used by the parser; holding + on to a reference to it is not a reliable way to keep a copy of the attributes. + To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* + object. + + +.. method:: ContentHandler.endElement(name) + + Signals the end of an element in non-namespace mode. + + The *name* parameter contains the name of the element type, just as with the + :meth:`startElement` event. + + +.. method:: ContentHandler.startElementNS(name, qname, attrs) + + Signals the start of an element in namespace mode. + + The *name* parameter contains the name of the element type as a ``(uri, + localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in + the source document, and the *attrs* parameter holds an instance of the + :class:`~xml.sax.xmlreader.AttributesNS` interface (see + :ref:`attributes-ns-objects`) + containing the attributes of the element. If no namespace is associated with + the element, the *uri* component of *name* will be ``None``. The object passed + as *attrs* may be re-used by the parser; holding on to a reference to it is not + a reliable way to keep a copy of the attributes. To keep a copy of the + attributes, use the :meth:`copy` method of the *attrs* object. + + Parsers may set the *qname* parameter to ``None``, unless the + ``feature_namespace_prefixes`` feature is activated. + + +.. method:: ContentHandler.endElementNS(name, qname) + + Signals the end of an element in namespace mode. + + The *name* parameter contains the name of the element type, just as with the + :meth:`startElementNS` method, likewise the *qname* parameter. + + +.. method:: ContentHandler.characters(content) + + Receive notification of character data. + + The Parser will call this method to report each chunk of character data. SAX + parsers may return all contiguous character data in a single chunk, or they may + split it into several chunks; however, all of the characters in any single event + must come from the same external entity so that the Locator provides useful + information. + + *content* may be a string or bytes instance; the ``expat`` reader module + always produces strings. + + .. note:: + + The earlier SAX 1 interface provided by the Python XML Special Interest Group + used a more Java-like interface for this method. Since most parsers used from + Python did not take advantage of the older interface, the simpler signature was + chosen to replace it. To convert old code to the new interface, use *content* + instead of slicing content with the old *offset* and *length* parameters. + + +.. method:: ContentHandler.ignorableWhitespace(whitespace) + + Receive notification of ignorable whitespace in element content. + + Validating Parsers must use this method to report each chunk of ignorable + whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating + parsers may also use this method if they are capable of parsing and using + content models. + + SAX parsers may return all contiguous whitespace in a single chunk, or they may + split it into several chunks; however, all of the characters in any single event + must come from the same external entity, so that the Locator provides useful + information. + + +.. method:: ContentHandler.processingInstruction(target, data) + + Receive notification of a processing instruction. + + The Parser will invoke this method once for each processing instruction found: + note that processing instructions may occur before or after the main document + element. + + A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a + text declaration (XML 1.0, section 4.3.1) using this method. + + +.. method:: ContentHandler.skippedEntity(name) + + Receive notification of a skipped entity. + + The Parser will invoke this method once for each entity skipped. Non-validating + processors may skip entities if they have not seen the declarations (because, + for example, the entity was declared in an external DTD subset). All processors + may skip external entities, depending on the values of the + ``feature_external_ges`` and the ``feature_external_pes`` properties. + + +.. _dtd-handler-objects: + +DTDHandler Objects +------------------ + +:class:`DTDHandler` instances provide the following methods: + + +.. method:: DTDHandler.notationDecl(name, publicId, systemId) + + Handle a notation declaration event. + + +.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) + + Handle an unparsed entity declaration event. + + +.. _entity-resolver-objects: + +EntityResolver Objects +---------------------- + + +.. method:: EntityResolver.resolveEntity(publicId, systemId) + + Resolve the system identifier of an entity and return either the system + identifier to read from as a string, or an InputSource to read from. The default + implementation returns *systemId*. + + +.. _sax-error-handler: + +ErrorHandler Objects +-------------------- + +Objects with this interface are used to receive error and warning information +from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that +implements this interface, then register the object with your +:class:`~xml.sax.xmlreader.XMLReader`, the parser +will call the methods in your object to report all warnings and errors. There +are three levels of errors available: warnings, (possibly) recoverable errors, +and unrecoverable errors. All methods take a :exc:`SAXParseException` as the +only parameter. Errors and warnings may be converted to an exception by raising +the passed-in exception object. + + +.. method:: ErrorHandler.error(exception) + + Called when the parser encounters a recoverable error. If this method does not + raise an exception, parsing may continue, but further document information + should not be expected by the application. Allowing the parser to continue may + allow additional errors to be discovered in the input document. + + +.. method:: ErrorHandler.fatalError(exception) + + Called when the parser encounters an error it cannot recover from; parsing is + expected to terminate when this method returns. + + +.. method:: ErrorHandler.warning(exception) + + Called when the parser presents minor warning information to the application. + Parsing is expected to continue when this method returns, and document + information will continue to be passed to the application. Raising an exception + in this method will cause parsing to end. + + +.. _lexical-handler-objects: + +LexicalHandler Objects +---------------------- +Optional SAX2 handler for lexical events. + +This handler is used to obtain lexical information about an XML +document. Lexical information includes information describing the +document encoding used and XML comments embedded in the document, as +well as section boundaries for the DTD and for any CDATA sections. +The lexical handlers are used in the same manner as content handlers. + +Set the LexicalHandler of an XMLReader by using the setProperty method +with the property identifier +``'http://xml.org/sax/properties/lexical-handler'``. + + +.. method:: LexicalHandler.comment(content) + + Reports a comment anywhere in the document (including the DTD and + outside the document element). + +.. method:: LexicalHandler.startDTD(name, public_id, system_id) + + Reports the start of the DTD declarations if the document has an + associated DTD. + +.. method:: LexicalHandler.endDTD() + + Reports the end of DTD declaration. + +.. method:: LexicalHandler.startCDATA() + + Reports the start of a CDATA marked section. + + The contents of the CDATA marked section will be reported through + the characters handler. + +.. method:: LexicalHandler.endCDATA() + + Reports the end of a CDATA marked section. diff --git a/Doc/library/zipimport.rst.bak b/Doc/library/zipimport.rst.bak new file mode 100644 index 000000000000000..62e1e47e980e179 --- /dev/null +++ b/Doc/library/zipimport.rst.bak @@ -0,0 +1,209 @@ +:mod:`zipimport` --- Import modules from Zip archives +===================================================== + +.. module:: zipimport + :synopsis: Support for importing Python modules from ZIP archives. + +.. moduleauthor:: Just van Rossum + +**Source code:** :source:`Lib/zipimport.py` + +-------------- + +This module adds the ability to import Python modules (:file:`\*.py`, +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not +needed to use the :mod:`zipimport` module explicitly; it is automatically used +by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths +to ZIP archives. + +Typically, :data:`sys.path` is a list of directory names as strings. This module +also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. +The ZIP archive can contain a subdirectory structure to support package imports, +and a path within the archive can be specified to only import from a +subdirectory. For example, the path :file:`example.zip/lib/` would only +import from the :file:`lib/` subdirectory within the archive. + +Any files may be present in the ZIP archive, but only files :file:`.py` and +:file:`.pyc` are available for import. ZIP import of dynamic modules +(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains +:file:`.py` files, Python will not attempt to modify the archive by adding the +corresponding :file:`.pyc` file, meaning that if a ZIP archive +doesn't contain :file:`.pyc` files, importing may be rather slow. + +.. versionchanged:: 3.8 + Previously, ZIP archives with an archive comment were not supported. + +.. seealso:: + + `PKZIP Application Note `_ + Documentation on the ZIP file format by Phil Katz, the creator of the format and + algorithms used. + + :pep:`273` - Import Modules from Zip Archives + Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 + follows the specification in :pep:`273`, but uses an implementation written by Just + van Rossum that uses the import hooks described in :pep:`302`. + + :mod:`importlib` - The implementation of the import machinery + Package providing the relevant protocols for all importers to + implement. + + +This module defines an exception: + +.. exception:: ZipImportError + + Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, + so it can be caught as :exc:`ImportError`, too. + + +.. _zipimporter-objects: + +zipimporter Objects +------------------- + +:class:`zipimporter` is the class for importing ZIP files. + +.. class:: zipimporter(archivepath) + + Create a new zipimporter instance. *archivepath* must be a path to a ZIP + file, or to a specific path within a ZIP file. For example, an *archivepath* + of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory + inside the ZIP file :file:`foo/bar.zip` (provided that it exists). + + :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP + archive. + + .. method:: create_module(spec) + + Implementation of :meth:`importlib.abc.Loader.create_module` that returns + :const:`None` to explicitly request the default semantics. + + .. versionadded:: 3.10 + + + .. method:: exec_module(module) + + Implementation of :meth:`importlib.abc.Loader.exec_module`. + + .. versionadded:: 3.10 + + + .. method:: find_loader(fullname, path=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_module(fullname, path=None) + + Search for a module specified by *fullname*. *fullname* must be the fully + qualified (dotted) module name. It returns the zipimporter instance itself + if the module was found, or :const:`None` if it wasn't. The optional + *path* argument is ignored---it's there for compatibility with the + importer protocol. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_spec(fullname, target=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. + + .. versionadded:: 3.10 + + + .. method:: get_code(fullname) + + Return the code object for the specified module. Raise + :exc:`ZipImportError` if the module couldn't be imported. + + + .. method:: get_data(pathname) + + Return the data associated with *pathname*. Raise :exc:`OSError` if the + file wasn't found. + + .. versionchanged:: 3.3 + :exc:`IOError` used to be raised instead of :exc:`OSError`. + + + .. method:: get_filename(fullname) + + Return the value ``__file__`` would be set to if the specified module + was imported. Raise :exc:`ZipImportError` if the module couldn't be + imported. + + .. versionadded:: 3.1 + + + .. method:: get_source(fullname) + + Return the source code for the specified module. Raise + :exc:`ZipImportError` if the module couldn't be found, return + :const:`None` if the archive does contain the module, but has no source + for it. + + + .. method:: is_package(fullname) + + Return ``True`` if the module specified by *fullname* is a package. Raise + :exc:`ZipImportError` if the module couldn't be found. + + + .. method:: load_module(fullname) + + Load the module specified by *fullname*. *fullname* must be the fully + qualified (dotted) module name. Returns the imported module on success, + raises :exc:`ZipImportError` on failure. + + .. deprecated:: 3.10 + + Use :meth:`exec_module` instead. + + .. attribute:: archive + + The file name of the importer's associated ZIP file, without a possible + subpath. + + + .. attribute:: prefix + + The subpath within the ZIP file where modules are searched. This is the + empty string for zipimporter objects which point to the root of the ZIP + file. + + The :attr:`archive` and :attr:`prefix` attributes, when combined with a + slash, equal the original *archivepath* argument given to the + :class:`zipimporter` constructor. + + +.. _zipimport-examples: + +Examples +-------- + +Here is an example that imports a module from a ZIP archive - note that the +:mod:`zipimport` module is not explicitly used. + +.. code-block:: shell-session + + $ unzip -l example.zip + Archive: example.zip + Length Date Time Name + -------- ---- ---- ---- + 8467 11-26-02 22:30 jwzthreading.py + -------- ------- + 8467 1 file + $ ./python + Python 2.3 (#1, Aug 1 2003, 19:54:32) + >>> import sys + >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path + >>> import jwzthreading + >>> jwzthreading.__file__ + 'example.zip/jwzthreading.py' diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak new file mode 100644 index 000000000000000..db71f061f14df45 --- /dev/null +++ b/Doc/whatsnew/3.10.rst.bak @@ -0,0 +1,1315 @@ +**************************** + What's New In Python 3.10 +**************************** + +:Release: |release| +:Date: |today| + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the bug/patch number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :issue:`12345`.) + + This saves the maintainer the effort of going through the Mercurial log + when researching a change. + +This article explains the new features in Python 3.10, compared to 3.9. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.10 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.10. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + +.. _whatsnew310-pep563: + +Parenthesized context managers +------------------------------ + +Using enclosing parentheses for continuation across multiple lines +in context managers is now supported. This allows formatting a long +collection of context managers in multiple lines in a similar way +as it was previously possible with import statements. For instance, +all these examples are now valid: + +.. code-block:: python + + with (CtxManager() as example): + ... + + with ( + CtxManager1(), + CtxManager2() + ): + ... + + with (CtxManager1() as example, + CtxManager2()): + ... + + with (CtxManager1(), + CtxManager2() as example): + ... + + with ( + CtxManager1() as example1, + CtxManager2() as example2 + ): + ... + +it is also possible to use a trailing comma at the end of the +enclosed group: + +.. code-block:: python + + with ( + CtxManager1() as example1, + CtxManager2() as example2, + CtxManager3() as example3, + ): + ... + +This new syntax uses the non LL(1) capacities of the new parser. +Check :pep:`617` for more details. + +(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou +in :issue:`12782` and :issue:`40334`.) + + +Better error messages in the parser +----------------------------------- + +When parsing code that contains unclosed parentheses or brackets the interpreter +now includes the location of the unclosed bracket of parentheses instead of displaying +*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. +For instance, consider the following code (notice the unclosed '{'): + +.. code-block:: python + + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, + some_other_code = foo() + +previous versions of the interpreter reported confusing places as the location of +the syntax error: + +.. code-block:: text + + File "example.py", line 3 + some_other_code = foo() + ^ + SyntaxError: invalid syntax + +but in Python3.10 a more informative error is emitted: + +.. code-block:: text + + File "example.py", line 1 + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + ^ + SyntaxError: '{' was never closed + + +In a similar way, errors involving unclosed string literals (single and triple +quoted) now point to the start of the string instead of reporting EOF/EOL. + +These improvements are inspired by previous work in the PyPy interpreter. + +(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in +:issue:`40176`.) + + +PEP 634: Structural Pattern Matching +------------------------------------ + +Structural pattern matching has been added in the form of a *match statement* +and *case statements* of patterns with associated actions. Patterns +consist of sequences, mappings, primitive data types as well as class instances. +Pattern matching enables programs to extract information from complex data types, +branch on the structure of data, and apply specific actions based on different +forms of data. + +Syntax and operations +~~~~~~~~~~~~~~~~~~~~~ + +The generic syntax of pattern matching is:: + + match subject: + case : + + case : + + case : + + case _: + + +A match statement takes an expression and compares its value to successive +patterns given as one or more case blocks. Specifically, pattern matching +operates by: + + 1. using data with type and shape (the ``subject``) + 2. evaluating the ``subject`` in the ``match`` statement + 3. comparing the subject with each pattern in a ``case`` statement + from top to bottom until a match is confirmed. + 4. executing the action associated with the pattern of the confirmed + match + 5. If an exact match is not confirmed, the last case, a wildcard ``_``, + if provided, will be used as the matching case. If an exact match is + not confirmed and a wildcard case does not exists, the entire match + block is a no-op. + +Declarative approach +~~~~~~~~~~~~~~~~~~~~ + +Readers may be aware of pattern matching through the simple example of matching +a subject (data object) to a literal (pattern) with the switch statement found +in C, Java or JavaScript (and many other languages). Often the switch statement +is used for comparison of an object/expression with case statements containing +literals. + +More powerful examples of pattern matching can be found in languages, such as +Scala and Elixir. With structural pattern matching, the approach is "declarative" and +explicitly states the conditions (the patterns) for data to match. + +While an "imperative" series of instructions using nested "if" statements +could be used to accomplish something similar to structural pattern matching, +it is less clear than the "declarative" approach. Instead the "declarative" +approach states the conditions to meet for a match and is more readable through +its explicit patterns. While structural pattern matching can be used in its +simplest form comparing a variable to a literal in a case statement, its +true value for Python lies in its handling of the subject's type and shape. + +Simple pattern: match to a literal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's look at this example as pattern matching in its simplest form: a value, +the subject, being matched to several literals, the patterns. In the example +below, ``status`` is the subject of the match statement. The patterns are +each of the case statements, where literals represent request status codes. +The associated action to the case is executed after a match:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something's wrong with the Internet" + +If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. +If the above function is passed a ``status`` of 500, the case statement with +``_`` will match as a wildcard, and "Something's wrong with the Internet" is +returned. +Note the last block: the variable name, ``_``, acts as a *wildcard* and insures +the subject will always match. The use of ``_`` is optional. + +You can combine several literals in a single pattern using ``|`` ("or"):: + + case 401 | 403 | 404: + return "Not allowed" + +Behavior without the wildcard +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If we modify the above example by removing the last case block, the example +becomes:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + +Without the use of ``_`` in a case statement, a match may not exist. If no +match exists, the behavior is a no-op. For example, if ``status`` of 500 is +passed, a no-op occurs. + +Patterns with a literal and variable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Patterns can look like unpacking assignments, and a pattern may be used to bind +variables. In this example, a data point can be unpacked to its x-coordinate +and y-coordinate:: + + # point is an (x, y) tuple + match point: + case (0, 0): + print("Origin") + case (0, y): + print(f"Y={y}") + case (x, 0): + print(f"X={x}") + case (x, y): + print(f"X={x}, Y={y}") + case _: + raise ValueError("Not a point") + +The first pattern has two literals, ``(0, 0)``, and may be thought of as an +extension of the literal pattern shown above. The next two patterns combine a +literal and a variable, and the variable *binds* a value from the subject +(``point``). The fourth pattern captures two values, which makes it +conceptually similar to the unpacking assignment ``(x, y) = point``. + +Patterns and classes +~~~~~~~~~~~~~~~~~~~~ + +If you are using classes to structure your data, you can use as a pattern +the class name followed by an argument list resembling a constructor. This +pattern has the ability to capture class attributes into variables:: + + class Point: + x: int + y: int + + def location(point): + match point: + case Point(x=0, y=0): + print("Origin is the point's location.") + case Point(x=0, y=y): + print(f"Y={y} and the point is on the y-axis.") + case Point(x=x, y=0): + print(f"X={x} and the point is on the x-axis.") + case Point(): + print("The point is located somewhere else on the plane.") + case _: + print("Not a point") + +Patterns with positional parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use positional parameters with some builtin classes that provide an +ordering for their attributes (e.g. dataclasses). You can also define a specific +position for attributes in patterns by setting the ``__match_args__`` special +attribute in your classes. If it's set to ("x", "y"), the following patterns +are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: + + Point(1, var) + Point(1, y=var) + Point(x=1, y=var) + Point(y=var, x=1) + +Nested patterns +~~~~~~~~~~~~~~~ + +Patterns can be arbitrarily nested. For example, if our data is a short +list of points, it could be matched like this:: + + match points: + case []: + print("No points in the list.") + case [Point(0, 0)]: + print("The origin is the only point in the list.") + case [Point(x, y)]: + print(f"A single point {x}, {y} is in the list.") + case [Point(0, y1), Point(0, y2)]: + print(f"Two points on the Y axis at {y1}, {y2} are in the list.") + case _: + print("Something else is found in the list.") + +Complex patterns and the wildcard +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To this point, the examples have used ``_`` alone in the last case statement. +A wildcard can be used in more complex patterns, such as ``('error', code, _)``. +For example:: + + match test_variable: + case ('warning', code, 40): + print("A warning has been received.") + case ('error', code, _): + print(f"An error {code} occured.") + +In the above case, ``test_variable`` will match for ('error', code, 100) and +('error', code, 800). + +Guard +~~~~~ + +We can add an ``if`` clause to a pattern, known as a "guard". If the +guard is false, ``match`` goes on to try the next case block. Note +that value capture happens before the guard is evaluated:: + + match point: + case Point(x, y) if x == y: + print(f"The point is located on the diagonal Y=X at {x}.") + case Point(x, y): + print(f"Point is not on the diagonal.") + +Other Key Features +~~~~~~~~~~~~~~~~~~ + +Several other key features: + +- Like unpacking assignments, tuple and list patterns have exactly the + same meaning and actually match arbitrary sequences. Technically, + the subject must be an instance of ``collections.abc.Sequence``. + Therefore, an important exception is that patterns don't match iterators. + Also, to prevent a common mistake, sequence patterns don't match strings. + +- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, + *rest)`` work similar to wildcards in unpacking assignments. The + name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence + of at least two items without binding the remaining items. + +- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the + ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence + patterns, extra keys are ignored. A wildcard ``**rest`` is also + supported. (But ``**_`` would be redundant, so it not allowed.) + +- Subpatterns may be captured using the ``as`` keyword:: + + case (Point(x1, y1), Point(x2, y2) as p2): ... + + This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, + and p2 to the entire second item of the subject. + +- Most literals are compared by equality. However, the singletons ``True``, + ``False`` and ``None`` are compared by identity. + +- Named constants may be used in patterns. These named constants must be + dotted names to prevent the constant from being interpreted as a capture + variable:: + + from enum import Enum + class Color(Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + match color: + case Color.RED: + print("I see red!") + case Color.GREEN: + print("Grass is green") + case Color.BLUE: + print("I'm feeling the blues :(") + +For the full specification see :pep:`634`. Motivation and rationale +are in :pep:`635`, and a longer tutorial is in :pep:`636`. + + +New Features Related to Type Annotations +======================================== + +This section covers major changes affecting :pep:`484` type annotations and +the :mod:`typing` module. + + +PEP 563: Postponed Evaluation of Annotations Becomes Default +------------------------------------------------------------ + +In Python 3.7, postponed evaluation of annotations was added, +to be enabled with a ``from __future__ import annotations`` +directive. In 3.10 this became the default behavior, even +without that future directive. With this being default, all +annotations stored in :attr:`__annotations__` will be strings. +If needed, annotations can be resolved at runtime using +:func:`typing.get_type_hints`. See :pep:`563` for a full +description. Also, the :func:`inspect.signature` will try to +resolve types from now on, and when it fails it will fall back to +showing the string annotations. (Contributed by Batuhan Taskaya +in :issue:`38605`.) + + +PEP 604: New Type Union Operator +-------------------------------- + +A new type union operator was introduced which enables the syntax ``X | Y``. +This provides a cleaner way of expressing 'either type X or type Y' instead of +using :data:`typing.Union`, especially in type hints (annotations). + +In previous versions of Python, to apply a type hint for functions accepting +arguments of multiple types, :data:`typing.Union` was used:: + + def square(number: Union[int, float]) -> Union[int, float]: + return number ** 2 + + +Type hints can now be written in a more succinct manner:: + + def square(number: int | float) -> int | float: + return number ** 2 + + +This new syntax is also accepted as the second argument to :func:`isinstance` +and :func:`issubclass`:: + + >>> isinstance(1, int | str) + True + +See :ref:`types-union` and :pep:`604` for more details. + +(Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) + + +PEP 612: Parameter Specification Variables +------------------------------------------ + +Two new options to improve the information provided to static type checkers for +:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. + +The first is the parameter specification variable. They are used to forward the +parameter types of one callable to another callable -- a pattern commonly +found in higher order functions and decorators. Examples of usage can be found +in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate +dependency of parameter types in such a precise manner. + +The second option is the new ``Concatenate`` operator. It's used in conjunction +with parameter specification variables to type annotate a higher order callable +which adds or removes parameters of another callable. Examples of usage can +be found in :class:`typing.Concatenate`. + +See :class:`typing.Callable`, :class:`typing.ParamSpec`, +:class:`typing.Concatenate` and :pep:`612` for more details. + +(Contributed by Ken Jin in :issue:`41559`.) + + +PEP 613: TypeAlias Annotation +----------------------------- + +:pep:`484` introduced the concept of type aliases, only requiring them to be +top-level unannotated assignments. This simplicity sometimes made it difficult +for type checkers to distinguish between type aliases and ordinary assignments, +especially when forward references or invalid types were involved. Compare:: + + StrCache = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to +declare type aliases more explicitly:: + + StrCache: TypeAlias = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +See :pep:`613` for more details. + +(Contributed by Mikhail Golubev in :issue:`41923`.) + + +Other Language Changes +====================== + +* The :class:`int` type has a new method :meth:`int.bit_count`, returning the + number of ones in the binary expansion of a given integer, also known + as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) + +* The views returned by :meth:`dict.keys`, :meth:`dict.values` and + :meth:`dict.items` now all have a ``mapping`` attribute that gives a + :class:`types.MappingProxyType` object wrapping the original + dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) + +* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used + to require that all the iterables have an equal length. + +* Builtin and extension functions that take integer arguments no longer accept + :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other + objects that can be converted to integers only with a loss (e.g. that have + the :meth:`~object.__int__` method but do not have the + :meth:`~object.__index__` method). + (Contributed by Serhiy Storchaka in :issue:`37999`.) + +* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will + correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. + (Contributed by Alex Shkop in :issue:`38302`.) + +* Assignment expressions can now be used unparenthesized within set literals + and set comprehensions, as well as in sequence indexes (but not slices). + +* Functions have a new ``__builtins__`` attribute which is used to look for + builtin symbols when a function is executed, instead of looking into + ``__globals__['__builtins__']``. The attribute is initialized from + ``__globals__["__builtins__"]`` if it exists, else from the current builtins. + (Contributed by Mark Shannon in :issue:`42990`.) + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + +argparse +-------- + +Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. +(Contributed by Raymond Hettinger in :issue:`9694`.) + +base64 +------ + +Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the +Base32 Encoding with Extended Hex Alphabet. + +codecs +------ + +Add a :func:`codecs.unregister` function to unregister a codec search function. +(Contributed by Hai Shi in :issue:`41842`.) + +collections.abc +--------------- + +The ``__args__`` of the :ref:`parameterized generic ` for +:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. +:class:`collections.abc.Callable` generic now flattens type parameters, similar +to what :data:`typing.Callable` currently does. This means that +``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of +``(int, str, str)``; previously this was ``([int, str], str)``. To allow this +change, :class:`types.GenericAlias` can now be subclassed, and a subclass will +be returned when subscripting the :class:`collections.abc.Callable` type. Note +that a :exc:`TypeError` may be raised for invalid forms of parameterizing +:class:`collections.abc.Callable` which may have passed silently in Python 3.9. +(Contributed by Ken Jin in :issue:`42195`.) + +contextlib +---------- + +Add a :func:`contextlib.aclosing` context manager to safely close async generators +and objects representing asynchronously released resources. +(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) + +Add asynchronous context manager support to :func:`contextlib.nullcontext`. +(Contributed by Tom Gringauz in :issue:`41543`.) + +curses +------ + +The extended color functions added in ncurses 6.1 will be used transparently +by :func:`curses.color_content`, :func:`curses.init_color`, +:func:`curses.init_pair`, and :func:`curses.pair_content`. A new function, +:func:`curses.has_extended_color_support`, indicates whether extended color +support is provided by the underlying ncurses library. +(Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) + +The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if +they are provided by the underlying curses library. +(Contributed by Zackery Spytz in :issue:`39273`.) + +.. _distutils-deprecated: + +distutils +--------- + +The entire ``distutils`` package is deprecated, to be removed in Python +3.12. Its functionality for specifying package builds has already been +completely replaced by third-party packages ``setuptools`` and +``packaging``, and most other commonly used APIs are available elsewhere +in the standard library (such as :mod:`platform`, :mod:`shutil`, +:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate +any other functionality from ``distutils``, and applications that are +using other functions should plan to make private copies of the code. +Refer to :pep:`632` for discussion. + +The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. +The ``bdist_wheel`` command is now recommended to distribute binary packages +on Windows. +(Contributed by Victor Stinner in :issue:`42802`.) + +doctest +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +encodings +--------- +:func:`encodings.normalize_encoding` now ignores non-ASCII characters. +(Contributed by Hai Shi in :issue:`39337`.) + +glob +---- + +Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and +:func:`~glob.iglob` which allow to specify the root directory for searching. +(Contributed by Serhiy Storchaka in :issue:`38144`.) + +inspect +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +Added *globalns* and *localns* parameters in :func:`~inspect.signature` and +:meth:`inspect.Signature.from_callable` to retrieve the annotations in given +local and global namespaces. +(Contributed by Batuhan Taskaya in :issue:`41960`.) + +linecache +--------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +os +-- + +Added :func:`os.cpu_count()` support for VxWorks RTOS. +(Contributed by Peixing Xin in :issue:`41440`.) + +Added a new function :func:`os.eventfd` and related helpers to wrap the +``eventfd2`` syscall on Linux. +(Contributed by Christian Heimes in :issue:`41001`.) + +Added :func:`os.splice()` that allows to move data between two file +descriptors without copying between kernel address space and user +address space, where one of the file descriptors must refer to a +pipe. (Contributed by Pablo Galindo in :issue:`41625`.) + +Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` +and :data:`~os.O_NOFOLLOW_ANY` for macOS. +(Contributed by Dong-hee Na in :issue:`43106`.) + +pathlib +------- + +Added slice support to :attr:`PurePath.parents `. +(Contributed by Joshua Cannon in :issue:`35498`) + +Added negative indexing support to :attr:`PurePath.parents +`. +(Contributed by Yaroslav Pankovych in :issue:`21041`) + +platform +-------- + +Added :func:`platform.freedesktop_os_release()` to retrieve operation system +identification from `freedesktop.org os-release +`_ standard file. +(Contributed by Christian Heimes in :issue:`28468`) + +py_compile +---------- + +Added ``--quiet`` option to command-line interface of :mod:`py_compile`. +(Contributed by Gregory Schevchenko in :issue:`38731`.) + +pyclbr +------ + +Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` +objects in the tree returned by :func:`pyclbr.readline` and +:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. +(Contributed by Aviral Srivastava in :issue:`38307`.) + +shelve +------ + +The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default +instead of :mod:`pickle` protocol ``3`` when creating shelves. +(Contributed by Zackery Spytz in :issue:`34204`.) + +site +---- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +socket +------ + +The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. +(Contributed by Christian Heimes in :issue:`42413`.) + +sys +--- + +Add :data:`sys.orig_argv` attribute: the list of the original command line +arguments passed to the Python executable. +(Contributed by Victor Stinner in :issue:`23427`.) + +Add :data:`sys.stdlib_module_names`, containing the list of the standard library +module names. +(Contributed by Victor Stinner in :issue:`42955`.) + +threading +--------- + +Added :func:`threading.gettrace` and :func:`threading.getprofile` to +retrieve the functions set by :func:`threading.settrace` and +:func:`threading.setprofile` respectively. +(Contributed by Mario Corchero in :issue:`42251`.) + +Add :data:`threading.__excepthook__` to allow retrieving the original value +of :func:`threading.excepthook` in case it is set to a broken or a different +value. +(Contributed by Mario Corchero in :issue:`42308`.) + +traceback +--------- + +The :func:`~traceback.format_exception`, +:func:`~traceback.format_exception_only`, and +:func:`~traceback.print_exception` functions can now take an exception object +as a positional-only argument. +(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +types +----- + +Reintroduced the :data:`types.EllipsisType`, :data:`types.NoneType` +and :data:`types.NotImplementedType` classes, providing a new set +of types readily interpretable by type checkers. +(Contributed by Bas van Beek in :issue:`41810`.) + +typing +------ + +For major changes, see `New Features Related to Type Annotations`_. + +The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` +and to match the behavior of static type checkers specified in the PEP. + +1. ``Literal`` now de-duplicates parameters. +2. Equality comparisons between ``Literal`` objects are now order independent. +3. ``Literal`` comparisons now respects types. For example, + ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is + now ``False``. To support this change, the internally used type cache now + supports differentiating types. +4. ``Literal`` objects will now raise a :exc:`TypeError` exception during + equality comparisons if one of their parameters are not :term:`immutable`. + Note that declaring ``Literal`` with mutable parameters will not throw + an error:: + + >>> from typing import Literal + >>> Literal[{0}] + >>> Literal[{0}] == Literal[{False}] + Traceback (most recent call last): + File "", line 1, in + TypeError: unhashable type: 'set' + +(Contributed by Yurii Karabas in :issue:`42345`.) + +unittest +-------- + +Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the +existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi +in :issue:`39385`.) + +urllib.parse +------------ + +Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +xml +--- + +Add a :class:`~xml.sax.handler.LexicalHandler` class to the +:mod:`xml.sax.handler` module. +(Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) + +zipimport +--------- +Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, +:meth:`zipimport.zipimporter.create_module`, and +:meth:`zipimport.zipimporter.exec_module`. +(Contributed by Brett Cannon in :issue:`42131`. + + +Optimizations +============= + +* Constructors :func:`str`, :func:`bytes` and :func:`bytearray` are now faster + (around 30--40% for small objects). + (Contributed by Serhiy Storchaka in :issue:`41334`.) + +* The :mod:`runpy` module now imports fewer modules. + The ``python3 -m module-name`` command startup time is 1.3x faster in + average. + (Contributed by Victor Stinner in :issue:`41006`.) + +* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It + is about 36% faster now for regular attributes and 44% faster for slots. + (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido + van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy + and MicroPython.) + +* When building Python with ``--enable-optimizations`` now + ``-fno-semantic-interposition`` is added to both the compile and link line. + This speeds builds of the Python interpreter created with ``--enable-shared`` + with ``gcc`` by up to 30%. See `this article + `_ + for more details. (Contributed by Victor Stinner and Pablo Galindo in + :issue:`38980`.) + + +* Function parameters and their annotations are no longer computed at runtime, + but rather at compilation time. They are stored as a tuple of strings at the + bytecode level. It is now around 2 times faster to create a function with + parameter annotations. (Contributed by Yurii Karabas and Inada Naoki + in :issue:`42202`) + +* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` + now sometimes use Crochemore & Perrin's "Two-Way" string searching + algorithm to avoid quadratic behavior on long strings. (Contributed + by Dennis Sweeney in :issue:`41972`) + +Deprecated +========== + +* Starting in this release, there will be a concerted effort to begin + cleaning up old import semantics that were kept for Python 2.7 + compatibility. Specifically, + :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` + (superseded by :meth:`~importlib.abc.Finder.find_spec`), + :meth:`~importlib.abc.Loader.load_module` + (superseded by :meth:`~importlib.abc.Loader.exec_module`), + :meth:`~importlib.abc.Loader.module_repr` (which the import system + takes care of for you), the ``__package__`` attribute + (superseded by ``__spec__.parent``), the ``__loader__`` attribute + (superseded by ``__spec__.loader``), and the ``__cached__`` attribute + (superseded by ``__spec__.cached``) will slowly be removed (as well + as other classes and methods in :mod:`importlib`). + :exc:`ImportWarning` and/or :exc:`DeprecationWarning` will be raised + as appropriate to help identify code which needs updating during + this transition. + +* The entire ``distutils`` namespace is deprecated, to be removed in + Python 3.12. Refer to the :ref:`module changes ` + section for more information. + +* Non-integer arguments to :func:`random.randrange` are deprecated. + The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. + (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) + +* The various ``load_module()`` methods of :mod:`importlib` have been + documented as deprecated since Python 3.6, but will now also trigger + a :exc:`DeprecationWarning`. Use + :meth:`~importlib.abc.Loader.exec_module` instead. + (Contributed by Brett Cannon in :issue:`26131`.) + +* :meth:`zimport.zipimporter.load_module` has been deprecated in + preference for :meth:`~zipimport.zipimporter.exec_module`. + (Contributed by Brett Cannon in :issue:`26131`.) + +* The use of :meth:`~importlib.abc.Loader.load_module` by the import + system now triggers an :exc:`ImportWarning` as + :meth:`~importlib.abc.Loader.exec_module` is preferred. + (Contributed by Brett Cannon in :issue:`26131`.) + +* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python + 3.3, when it was made an alias to :class:`str`. It is now deprecated, + scheduled for removal in Python 3.12. + (Contributed by Erlend E. Aasland in :issue:`42264`.) + +* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now + deprecated, scheduled for removal in Python 3.12. Its use is strongly + discouraged by the SQLite3 documentation. See `the SQLite3 docs + `_ for more details. + If shared cache must be used, open the database in URI mode using the + ``cache=shared`` query parameter. + (Contributed by Erlend E. Aasland in :issue:`24464`.) + + +Removed +======= + +* Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, + ``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and + ``__rdivmod__`` of the :class:`complex` class. They always raised + a :exc:`TypeError`. + (Contributed by Serhiy Storchaka in :issue:`41974`.) + +* The ``ParserBase.error()`` method from the private and undocumented ``_markupbase`` + module has been removed. :class:`html.parser.HTMLParser` is the only subclass of + ``ParserBase`` and its ``error()`` implementation has already been removed in + Python 3.5. + (Contributed by Berker Peksag in :issue:`31844`.) + +* Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal + PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was + moved to the internal C API. + (Contributed by Victor Stinner in :issue:`42157`.) + +* Removed the ``parser`` module, which was deprecated in 3.9 due to the + switch to the new PEG parser, as well as all the C source and header files + that were only being used by the old parser, including ``node.h``, ``parser.h``, + ``graminit.h`` and ``grammar.h``. + +* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, + :c:func:`PyParser_SimpleParseStringFlagsFilename`, + :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` + that were deprecated in 3.9 due to the switch to the new PEG parser. + +* Removed the ``formatter`` module, which was deprecated in Python 3.4. + It is somewhat obsolete, little used, and not tested. It was originally + scheduled to be removed in Python 3.6, but such removals were delayed until + after Python 2.7 EOL. Existing users should copy whatever classes they use + into their code. + (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) + +* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless + now due to the _warnings module was converted to a builtin module in 2.6. + (Contributed by Hai Shi in :issue:`42599`.) + +* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from + the :mod:`collections` module. + (Contributed by Victor Stinner in :issue:`37324`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. The motivation behind this change is multifold: + + 1. This simplifies the high-level API. + 2. The functions in the high-level API have been implicitly getting the + current thread's running event loop since Python 3.7. There isn't a need to + pass the event loop to the API in most normal use cases. + 3. Event loop passing is error-prone especially when dealing with loops + running in different threads. + + Note that the low-level API will still accept ``loop``. + See `Changes in the Python API`_ for examples of how to replace existing code. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + + +Porting to Python 3.10 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in the Python API +------------------------- + +* The *etype* parameters of the :func:`~traceback.format_exception`, + :func:`~traceback.format_exception_only`, and + :func:`~traceback.print_exception` functions in the :mod:`traceback` module + have been renamed to *exc*. + (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +* :mod:`atexit`: At Python exit, if a callback registered with + :func:`atexit.register` fails, its exception is now logged. Previously, only + some exceptions were logged, and the last exception was always silently + ignored. + (Contributed by Victor Stinner in :issue:`42639`.) + +* :class:`collections.abc.Callable` generic now flattens type parameters, similar + to what :data:`typing.Callable` currently does. This means that + ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of + ``(int, str, str)``; previously this was ``([int, str], str)``. Code which + accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account + for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms + of parameterizing :class:`collections.abc.Callable` which may have passed + silently in Python 3.9. + (Contributed by Ken Jin in :issue:`42195`.) + +* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` + instead of :exc:`DeprecationWarning` if the given parameter will not fit in + a 16-bit unsigned integer. + (Contributed by Erlend E. Aasland in :issue:`42393`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. + + A coroutine that currently look like this:: + + async def foo(loop): + await asyncio.sleep(1, loop=loop) + + Should be replaced with this:: + + async def foo(): + await asyncio.sleep(1) + + If ``foo()`` was specifically designed *not* to run in the current thread's + running event loop (e.g. running in another thread's event loop), consider + using :func:`asyncio.run_coroutine_threadsafe` instead. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + +* The :data:`types.FunctionType` constructor now inherits the current builtins + if the *globals* dictionary has no ``"__builtins__"`` key, rather than using + ``{"None": None}`` as builtins: same behavior as :func:`eval` and + :func:`exec` functions. Defining a function with ``def function(...): ...`` + in Python is not affected, globals cannot be overriden with this syntax: it + also inherits the current builtins. + (Contributed by Victor Stinner in :issue:`42990`.) + +CPython bytecode changes +======================== + +* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations + instead of dictionary. + (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) + +Build Changes +============= + +* The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required + to build Python. + (Contributed by Victor Stinner in :issue:`36020`.) + +* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev + and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) + +* The :mod:`atexit` module must now always be built as a built-in module. + (Contributed by Victor Stinner in :issue:`42639`.) + +* Added ``--disable-test-modules`` option to the ``configure`` script: + don't build nor install test modules. + (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) + +* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If + specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` + wheel packages in this directory: if both are present, these wheel packages + are used instead of ensurepip bundled wheel packages. + + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + +* Add a new configure ``--without-static-libpython`` option to not build the + ``libpythonMAJOR.MINOR.a`` static library and not install the ``python.o`` + object file. + + (Contributed by Victor Stinner in :issue:`43103`.) + +* The ``configure`` script now uses the ``pkg-config`` utility, if available, + to detect the location of Tcl/Tk headers and libraries. As before, those + locations can be explicitly specified with the ``--with-tcltk-includes`` + and ``--with-tcltk-libs`` configuration options. + (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) + + +C API Changes +============= + +New Features +------------ + +* The result of :c:func:`PyNumber_Index` now always has exact type :class:`int`. + Previously, the result could have been an instance of a subclass of ``int``. + (Contributed by Serhiy Storchaka in :issue:`40792`.) + +* Add a new :c:member:`~PyConfig.orig_argv` member to the :c:type:`PyConfig` + structure: the list of the original command line arguments passed to the + Python executable. + (Contributed by Victor Stinner in :issue:`23427`.) + +* The :c:func:`PyDateTime_DATE_GET_TZINFO` and + :c:func:`PyDateTime_TIME_GET_TZINFO` macros have been added for accessing + the ``tzinfo`` attributes of :class:`datetime.datetime` and + :class:`datetime.time` objects. + (Contributed by Zackery Spytz in :issue:`30155`.) + +* Add a :c:func:`PyCodec_Unregister` function to unregister a codec + search function. + (Contributed by Hai Shi in :issue:`41842`.) + +* The :c:func:`PyIter_Send` function was added to allow + sending value into iterator without raising ``StopIteration`` exception. + (Contributed by Vladimir Matveev in :issue:`41756`.) + +* Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. + (Contributed by Alex Gaynor in :issue:`41784`.) + +* Added :c:func:`PyModule_AddObjectRef` function: similar to + :c:func:`PyModule_AddObject` but don't steal a reference to the value on + success. + (Contributed by Victor Stinner in :issue:`1635741`.) + +* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the + reference count of an object and return the object. + (Contributed by Victor Stinner in :issue:`42262`.) + +* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` + functions now accept a single class as the *bases* argument. + (Contributed by Serhiy Storchaka in :issue:`42423`.) + +* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` + slot. + (Contributed by Hai Shi in :issue:`41832`.) + +* The :c:func:`PyType_GetSlot` function can accept static types. + (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) + +* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an + object is an instance of :class:`set` but not an instance of a subtype. + (Contributed by Pablo Galindo in :issue:`43277`.) + +Porting to Python 3.10 +---------------------- + +* The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use + :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use + ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. + See :ref:`Parsing arguments and building values + ` and the :pep:`353`. + (Contributed by Victor Stinner in :issue:`40943`.) + +* Since :c:func:`Py_REFCNT()` is changed to the inline static function, + ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: + see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward + compatibility, this macro can be used:: + + #if PY_VERSION_HEX < 0x030900A4 + # define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) + #endif + + (Contributed by Victor Stinner in :issue:`39573`.) + +* Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed + for historical reason. It is no longer allowed. + (Contributed by Victor Stinner in :issue:`40839`.) + +* ``PyUnicode_FromUnicode(NULL, size)`` and ``PyUnicode_FromStringAndSize(NULL, size)`` + raise ``DeprecationWarning`` now. Use :c:func:`PyUnicode_New` to allocate + Unicode object without initial data. + (Contributed by Inada Naoki in :issue:`36346`.) + +* The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API + ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. + (Contributed by Victor Stinner in :issue:`42157`.) + +* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, + :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and + :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before + :c:func:`Py_Initialize` (before Python is initialized). Use the new + :ref:`Python Initialization Configuration API ` to get the + :ref:`Python Path Configuration. `. + (Contributed by Victor Stinner in :issue:`42260`.) + +* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and + :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. + For example, ``x = PyList_SET_ITEM(a, b, c)`` and + ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents + bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. + (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) + +* The non-limited API files ``odictobject.h``, ``parser_interface.h``, + ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, + ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` + directory. These files must not be included directly, as they are already + included in ``Python.h``: :ref:`Include Files `. If they have + been included directly, consider including ``Python.h`` instead. + (Contributed by Nicholas Sim in :issue:`35134`) + +Deprecated +---------- + +* The ``PyUnicode_InternImmortal()`` function is now deprecated + and will be removed in Python 3.12: use :c:func:`PyUnicode_InternInPlace` + instead. + (Contributed by Victor Stinner in :issue:`41692`.) + +Removed +------- + +* ``PyObject_AsCharBuffer()``, ``PyObject_AsReadBuffer()``, ``PyObject_CheckReadBuffer()``, + and ``PyObject_AsWriteBuffer()`` are removed. Please migrate to new buffer protocol; + :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release`. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. + (Contributed by Inada Naoki in :issue:`41123`.) + + * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or + :c:macro:`PyUnicode_GET_LENGTH` + * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or + :c:func:`PyUnicode_FromFormat` + * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use + :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` + * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` + * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` + * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use + :c:func:`PyUnicode_FindChar` + +* Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``PyLong_FromUnicode()``. Please migrate to :c:func:`PyLong_FromUnicodeObject`. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``PyUnicode_AsUnicodeCopy()``. Please use :c:func:`PyUnicode_AsUCS4Copy` or + :c:func:`PyUnicode_AsWideCharString` + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``_Py_CheckRecursionLimit`` variable: it has been replaced by + ``ceval.recursion_limit`` of the :c:type:`PyInterpreterState` structure. + (Contributed by Victor Stinner in :issue:`41834`.) + +* Removed undocumented macros ``Py_ALLOW_RECURSION`` and + ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the + :c:type:`PyInterpreterState` structure. + (Contributed by Serhiy Storchaka in :issue:`41936`.) + +* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing + Python already implicitly installs signal handlers: see + :c:member:`PyConfig.install_signal_handlers`. + (Contributed by Victor Stinner in :issue:`41713`.) From 32cbefb60cd9dfa25b480f7659c01576c03af072 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 10:37:20 +0100 Subject: [PATCH 04/24] Update test_locks.py --- Lib/test/test_asyncio/test_locks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 42ef44db2d37589..26c16b3850d2960 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,8 +11,8 @@ STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?(, count:\d+/\d+)?(, state:^-?\d)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier object + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -71,7 +71,7 @@ def test_lock_doesnt_accept_loop_parameter(self): cls(loop=self.loop) # Barrier object has a positional paramater - # so check alone + # so test alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, From e5002bc7d677f8add141268a926bf9ee45da9dfd Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:09:54 +0100 Subject: [PATCH 05/24] Update locks.py Add comment when testing self._state Change if to while instruction in _block method --- Lib/asyncio/locks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 4bc69100ed225d8..e7f2561468e8cf1 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -452,7 +452,7 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._blocking = Event() # used block new tasks while waiting tasks are draining or broken self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken @@ -491,7 +491,7 @@ async def wait(self): # Block until the barrier is ready for us, or raise an exception # if it is broken. async def _block (self): - if self._state in (-1, 1): + while self._state in (-1, 1): # It is draining or resetting, wait until done await self._blocking.wait() @@ -522,7 +522,7 @@ def _release(self): async def _wait(self): await self._waiting.wait() # no timeout so - if self._state < 0: + if self._state < 0: # resetting or broken raise BrokenBarrierError assert self._state == 1, repr(self) @@ -530,9 +530,9 @@ async def _wait(self): # waiting for the barrier to drain. def _exit(self): if self._count == 0: - if self._state == 1: + if self._state == 1: # draining self._state = 0 - elif self._state == -1: + elif self._state == -1: # resetting self._state = 0 self._waiting.clear() self._blocking.set() @@ -544,12 +544,12 @@ def reset(self): raised. """ if self._count > 0: - if self._state in (0, 1): - #reset the barrier, waking up tasks + if self._state in (0, 1): # filling or draining + # reset the barrier, waking up tasks self._state = -1 elif self._state == -2: - #was broken, set it to reset state - #which clears when the last tasks exits + # was broken, set it to reset state + # which clears when the last tasks exits self._state = -1 self._waiting.set() self._blocking.clear() From b12599c21518b32da57605a32d12d71ff5d0b936 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:13:24 +0100 Subject: [PATCH 06/24] Update test_locks.py Refactoring all test names of BarrierTests to be more readable Add a method cancel_coros Refactoring and simplifyiing some tests Correction of RGX_REPR --- Lib/test/test_asyncio/test_locks.py | 253 +++++++++++++++++----------- 1 file changed, 155 insertions(+), 98 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 26c16b3850d2960..ec90698b6155aad 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -12,7 +12,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # new line dedicated to repr barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -71,7 +71,7 @@ def test_lock_doesnt_accept_loop_parameter(self): cls(loop=self.loop) # Barrier object has a positional paramater - # so test alone + # so tests alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, @@ -954,6 +954,11 @@ async def run_coros(self, n, coro): res = await asyncio.gather(*tasks) return res, tasks + def cancel_coros(self): + for task in asyncio.all_tasks(self.loop): + task.cancel() + test_utils.run_briefly(self.loop) + def test_repr(self): async def coro(): try: @@ -963,7 +968,7 @@ async def coro(): else: return await asyncio.sleep(1, True) - self.N = 999 + self.N = 1011001 barrier = asyncio.Barrier(self.N) self.assertTrue("[unset," in repr(barrier)) self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) @@ -998,23 +1003,24 @@ async def coro(): self.assertTrue(f"0/{self.N}" in repr(barrier)) self.assertTrue(repr(barrier).endswith(':-2]>')) + self.cancel_coros() + def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait_more(self): - async def coro(result): - r = await barrier.wait() - result.append(r) - + def test_wait_task_once(self): + self.N = 1 barrier = asyncio.Barrier(self.N) - results = [] - _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) - self.assertEqual(sum(results), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + r1 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(r1, r2) self.assertFalse(barrier.broken) - def test_wait_task_multi(self): + def test_wait_task_by_task(self): self.N = 3 barrier = asyncio.Barrier(self.N) self.assertEqual(barrier.n_waiting, 0) @@ -1029,144 +1035,181 @@ def test_wait_task_multi(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_task_one(self): - self.N = 1 - barrier = asyncio.Barrier(self.N) - r1 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - r2 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - self.assertEqual(r1, r2) - self.assertFalse(barrier.broken) - - def test_wait_step_by_step(self): - async def coro(result, value): + def test_wait_tasks_twice(self): + results = [] + async def coro(): await barrier.wait() - result.append(value) + results.append(True) + await barrier.wait() + results.append(False) - results = [] barrier = asyncio.Barrier(self.N) - value = 1 - tasks = [] - for n in range(self.N-1): - tasks.append(self.loop.create_task(coro(results, value))) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, n+1) - - tasks.append(self.loop.create_task(coro(results, value))) - test_utils.run_briefly(self.loop) - self.assertEqual(results, [value]* self.N) - self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(all(t.done() for t in tasks)) - self.assertTrue(all(t.result() is None for t in tasks)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results), self.N*2) + self.assertEqual(results.count(True), self.N) + self.assertEqual(results.count(False), self.N) self.assertFalse(barrier.broken) + self.assertEqual(barrier.n_waiting, 0) - def test_wait_multi_return(self): + def test_wait_return_from_wait(self): results1 = [] results2 = [] - results3 = [] async def coro(): await barrier.wait() results1.append(True) i = await barrier.wait() results2.append(True) - results3.append(i) return i barrier = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + res = self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertEqual(len(results1), self.N) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) self.assertTrue(all(results2)) - self.assertEqual(sum(res), sum(results3)) + self.assertEqual(sum(res[0]), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) - def test_barrier(self, multipass=1, nn=5): - results = [[] for _ in range(multipass)] + + def test_wait_repeat_more(self, more=5): async def coro(result, value): ret = await barrier.wait() result.append(value) - return ret - barrier = asyncio.Barrier(nn) - for i in range(multipass): - _ = [self.loop.create_task(coro(results[i], value)) for value in range(nn)] + results = [[] for _ in range(more)] + barrier = asyncio.Barrier(self.N) + for i in range(more): + _ = [self.loop.create_task(coro(results[i], value)) for value in range(self.N)] test_utils.run_briefly(self.loop) - self.assertEqual(len(results[i]), nn) - self.assertEqual(sum(results[i]), sum(range(nn))) + self.assertEqual(len(results[i]), self.N) + self.assertEqual(sum(results[i]), sum(range(self.N))) if i > 0: self.assertEqual(sum(results[i]), sum(results[i-1])) self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_state_draining(self): + result = [] + state = [] + async def coro(): + i = await barrier.wait() + if i == 0: + result.append(True) + # Do we need to add a Barrie.draining() property ? + state.append(repr(barrier).endswith("state:1]>")) + rest = self.N-1 + nb_tasks = self.N + rest + barrier = asyncio.Barrier(self.N) + for _ in range(nb_tasks): + self.loop.create_task(coro()) + for _ in range((nb_tasks//self.N)+1): + test_utils.run_briefly(self.loop) + self.assertEqual(len(result), nb_tasks//self.N) + self.assertTrue(all(result)) + self.assertTrue(all(state)) + self.assertEqual(barrier.n_waiting, rest) self.assertFalse(barrier.broken) - def test_barrier_multipass(self): - self.test_barrier(10) + self.cancel_coros() - def test_action_callback(self): - async def coro(result, value): - ret = await barrier.wait() - result.append(value) - return ret + def test_wait_tasks_cancel_one_task(self): + results = [] + async def coro(): + await barrier.wait() + results.append(True) - result = [] - result1 = [] - value = 1 - barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) - _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + self.N=3 + barrier = asyncio.Barrier(self.N) + t1 = self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) - self.assertEqual(len(result1), self.N) - self.assertTrue(all(result1)) - self.assertEqual(len(result), self.N) - self.assertEqual(sum(result), self.N*value) + t1.cancel() + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(len(results), self.N) + self.assertTrue(all(results)) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_action_callback_n(self): - async def coro(result, value): + def test_wait_action_callback(self): + result = [] + async def coro(): ret = await barrier.wait() - result.append(value) - return ret + result.append(True) - result = [] result1 = [] - value = 1 + self.N = 2 barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(len(result1), 1) self.assertTrue(result1[0]) - self.assertEqual(len(result), self.N) - self.assertEqual(sum(result), value*self.N) + self.assertEqual(len(result), 2) + self.assertTrue(all(result)) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_action_callback_more(self): + result = [] + async def coro(): + ret = await barrier.wait() + result.append(True) + + result1 = [] + + self.N = 2 + more = 3 + barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) + for _ in range(more): + _ = [self.loop.create_task(coro()) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), more) + self.assertTrue(all(result1)) + self.assertEqual(len(result), self.N*more) + self.assertTrue(all(result)) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_action_callback_error(self): + def test_wait_action_callback_error_broken(self): + ERROR = ZeroDivisionError results1 = [] results2 = [] - results3 = [] - def raise_except(e): - raise e - - async def coro(e): + async def coro(): try: ret = await barrier.wait() - except e: + except ERROR: results1.append(False) except: results2.append(True) - else: - results3.append(None) - barrier = asyncio.Barrier(self.N, lambda: raise_except(ZeroDivisionError)) - _ = [self.loop.create_task(coro(ZeroDivisionError)) for _ in range(self.N)] + def raise_except(): + raise ERROR + + barrier = asyncio.Barrier(self.N, lambda: raise_except()) + _ = [self.loop.create_task(coro()) for _ in range(self.N)] test_utils.run_briefly(self.loop) self.assertEqual(len(results1), 1) self.assertFalse(results1[0]) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) - self.assertEqual(len(results3), 0) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) def test_reset(self): @@ -1196,16 +1239,21 @@ async def coro2(): self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) - def test_reset_multi_wait(self): + def test_reset_and_wait(self): results1 = [] results2 = [] results3 = [] - alls = [] + count = 0 async def coro(): + nonlocal count + i = await barrier1.wait() - alls.append(i) - if len(alls) == self.N: # i == self.N//2: + count += 1 + if count == self.N: # i == self.N//2: barrier.reset() else: try: @@ -1220,13 +1268,16 @@ async def coro(): barrier = asyncio.Barrier(self.N) barrier1 = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) - def test_abort(self): + def test_abort_broken(self): results1 = [] results2 = [] async def coro(): @@ -1242,10 +1293,13 @@ async def coro(): barrier.abort() barrier = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) + self.assertEqual(barrier.n_waiting, 0) + + self.assertTrue(barrier.broken) def test_abort_and_reset(self): results1 = [] @@ -1275,11 +1329,14 @@ async def coro(): barrier = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) if __name__ == '__main__': From f12123e8c7bd56ca1835f042861272e11e4c5885 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:14:05 +0100 Subject: [PATCH 07/24] Update from patch after run python.bat Tools\scripts\patchcheck.py --- Doc/whatsnew/3.10.rst.bak | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak index db71f061f14df45..c4c282e5a04eae9 100644 --- a/Doc/whatsnew/3.10.rst.bak +++ b/Doc/whatsnew/3.10.rst.bak @@ -673,9 +673,16 @@ When a module does not define ``__loader__``, fall back to ``__spec__.loader``. encodings --------- + :func:`encodings.normalize_encoding` now ignores non-ASCII characters. (Contributed by Hai Shi in :issue:`39337`.) +gc +-- + +Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and +:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) + glob ---- @@ -683,6 +690,19 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and :func:`~glob.iglob` which allow to specify the root directory for searching. (Contributed by Serhiy Storchaka in :issue:`38144`.) +importlib.metadata +------------------ + +Feature parity with ``importlib_metadata`` 3.7. + +:func:`importlib.metadata.entry_points` now provides a nicer experience +for selecting entry points by group and name through a new +:class:`importlib.metadata.EntryPoints` class. + +Added :func:`importlib.metadata.packages_distributions` for resolving +top-level Python modules and packages to their +:class:`importlib.metadata.Distribution`. + inspect ------- @@ -781,6 +801,13 @@ Add :data:`sys.stdlib_module_names`, containing the list of the standard library module names. (Contributed by Victor Stinner in :issue:`42955`.) +_thread +------- + +:func:`_thread.interrupt_main` now takes an optional signal number to +simulate (the default is still :data:`signal.SIGINT`). +(Contributed by Antoine Pitrou in :issue:`43356`.) + threading --------- @@ -1203,6 +1230,11 @@ New Features object is an instance of :class:`set` but not an instance of a subtype. (Contributed by Pablo Galindo in :issue:`43277`.) +* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number + to simulate. + (Contributed by Antoine Pitrou in :issue:`43356`.) + + Porting to Python 3.10 ---------------------- From 9ef71090b995c6013f9d7ed586d1862552bf4727 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 31 Mar 2021 16:24:52 +0200 Subject: [PATCH 08/24] remove cpython/doc/*.rst.bak --- Doc/c-api/exceptions.rst.bak | 1054 ------------- Doc/c-api/memory.rst.bak | 642 -------- Doc/faq/design.rst.bak | 759 --------- Doc/howto/descriptor.rst.bak | 1575 ------------------- Doc/library/_thread.rst.bak | 215 --- Doc/library/ast.rst.bak | 1993 ------------------------ Doc/library/asyncio-api-index.rst.bak | 221 --- Doc/library/collections.rst.bak | 1278 --------------- Doc/library/dataclasses.rst.bak | 595 ------- Doc/library/gc.rst.bak | 322 ---- Doc/library/importlib.metadata.rst.bak | 262 ---- Doc/library/logging.rst.bak | 1350 ---------------- Doc/library/readline.rst.bak | 361 ----- Doc/library/sqlite3.rst.bak | 1094 ------------- Doc/library/statistics.rst.bak | 879 ----------- Doc/library/tempfile.rst.bak | 374 ----- Doc/library/venv.rst.bak | 496 ------ Doc/library/xml.sax.handler.rst.bak | 463 ------ Doc/library/zipimport.rst.bak | 209 --- Doc/whatsnew/3.10.rst.bak | 1347 ---------------- 20 files changed, 15489 deletions(-) delete mode 100644 Doc/c-api/exceptions.rst.bak delete mode 100644 Doc/c-api/memory.rst.bak delete mode 100644 Doc/faq/design.rst.bak delete mode 100644 Doc/howto/descriptor.rst.bak delete mode 100644 Doc/library/_thread.rst.bak delete mode 100644 Doc/library/ast.rst.bak delete mode 100644 Doc/library/asyncio-api-index.rst.bak delete mode 100644 Doc/library/collections.rst.bak delete mode 100644 Doc/library/dataclasses.rst.bak delete mode 100644 Doc/library/gc.rst.bak delete mode 100644 Doc/library/importlib.metadata.rst.bak delete mode 100644 Doc/library/logging.rst.bak delete mode 100644 Doc/library/readline.rst.bak delete mode 100644 Doc/library/sqlite3.rst.bak delete mode 100644 Doc/library/statistics.rst.bak delete mode 100644 Doc/library/tempfile.rst.bak delete mode 100644 Doc/library/venv.rst.bak delete mode 100644 Doc/library/xml.sax.handler.rst.bak delete mode 100644 Doc/library/zipimport.rst.bak delete mode 100644 Doc/whatsnew/3.10.rst.bak diff --git a/Doc/c-api/exceptions.rst.bak b/Doc/c-api/exceptions.rst.bak deleted file mode 100644 index 4e99a0167a632d7..000000000000000 --- a/Doc/c-api/exceptions.rst.bak +++ /dev/null @@ -1,1054 +0,0 @@ -.. highlight:: c - - -.. _exceptionhandling: - -****************** -Exception Handling -****************** - -The functions described in this chapter will let you handle and raise Python -exceptions. It is important to understand some of the basics of Python -exception handling. It works somewhat like the POSIX :c:data:`errno` variable: -there is a global indicator (per thread) of the last error that occurred. Most -C API functions don't clear this on success, but will set it to indicate the -cause of the error on failure. Most C API functions also return an error -indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` -if they return an integer (exception: the :c:func:`PyArg_\*` functions -return ``1`` for success and ``0`` for failure). - -Concretely, the error indicator consists of three object pointers: the -exception's type, the exception's value, and the traceback object. Any -of those pointers can be ``NULL`` if non-set (although some combinations are -forbidden, for example you can't have a non-``NULL`` traceback if the exception -type is ``NULL``). - -When a function must fail because some function it called failed, it generally -doesn't set the error indicator; the function it called already set it. It is -responsible for either handling the error and clearing the exception or -returning after cleaning up any resources it holds (such as object references or -memory allocations); it should *not* continue normally if it is not prepared to -handle the error. If returning due to an error, it is important to indicate to -the caller that an error has been set. If the error is not handled or carefully -propagated, additional calls into the Python/C API may not behave as intended -and may fail in mysterious ways. - -.. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. - The former corresponds to an exception that is not yet caught (and is - therefore still propagating), while the latter returns an exception after - it is caught (and has therefore stopped propagating). - - -Printing and clearing -===================== - - -.. c:function:: void PyErr_Clear() - - Clear the error indicator. If the error indicator is not set, there is no - effect. - - -.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) - - Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``, in that case no traceback is - printed and the Python process will exit with the error code specified by - the ``SystemExit`` instance. - - Call this function **only** when the error indicator is set. Otherwise it - will cause a fatal error! - - If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, - :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the - type, value and traceback of the printed exception, respectively. - - -.. c:function:: void PyErr_Print() - - Alias for ``PyErr_PrintEx(1)``. - - -.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) - - Call :func:`sys.unraisablehook` using the current exception and *obj* - argument. - - This utility function prints a warning message to ``sys.stderr`` when an - exception has been set but it is impossible for the interpreter to actually - raise the exception. It is used, for example, when an exception occurs in an - :meth:`__del__` method. - - The function is called with a single argument *obj* that identifies the context - in which the unraisable exception occurred. If possible, - the repr of *obj* will be printed in the warning message. - - An exception must be set when calling this function. - - -Raising exceptions -================== - -These functions help you set the current thread's error indicator. -For convenience, some of these functions will always return a -``NULL`` pointer for use in a ``return`` statement. - - -.. c:function:: void PyErr_SetString(PyObject *type, const char *message) - - This is the most common way to set the error indicator. The first argument - specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. - The second argument is an error message; it is decoded from ``'utf-8``'. - - -.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) - - This function is similar to :c:func:`PyErr_SetString` but lets you specify an - arbitrary Python object for the "value" of the exception. - - -.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - - This function sets the error indicator and returns ``NULL``. *exception* - should be a Python exception class. The *format* and subsequent - parameters help format the error message; they have the same meaning and - values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded - string. - - -.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) - - Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather - than a variable number of arguments. - - .. versionadded:: 3.5 - - -.. c:function:: void PyErr_SetNone(PyObject *type) - - This is a shorthand for ``PyErr_SetObject(type, Py_None)``. - - -.. c:function:: int PyErr_BadArgument() - - This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where - *message* indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use. - - -.. c:function:: PyObject* PyErr_NoMemory() - - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` - so an object allocation function can write ``return PyErr_NoMemory();`` when it - runs out of memory. - - -.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - - .. index:: single: strerror() - - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable :c:data:`errno`. It constructs a - tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`strerror`), - and then calls ``PyErr_SetObject(type, object)``. On Unix, when the - :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, - this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns ``NULL``, so a wrapper - function around a system call can write ``return PyErr_SetFromErrno(type);`` - when the system call returns an error. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) - - Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as - a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`filename` attribute of the - exception instance. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second - filename object, for raising errors when a function that takes two filenames - fails. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the :term:`filesystem - encoding and error handler`. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - - This is a convenience function to raise :exc:`WindowsError`. If called with - *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` - is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve - the Windows description of error code given by *ierr* or :c:func:`GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns ``NULL``. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter - specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) - - Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, - but accepts a second filename object. - - .. availability:: Windows. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional - parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) - - This is a convenience function to raise :exc:`ImportError`. *msg* will be - set as the exception's message string. *name* and *path*, both of which can - be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` - and ``path`` attributes. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) - - Set file, line, and offset information for the current exception. If the - current exception is not a :exc:`SyntaxError`, then it sets additional - attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. - - .. versionadded:: 3.4 - - -.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) - - Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the :term:`filesystem encoding and error handler`. - - .. versionadded:: 3.2 - - -.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) - - Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is - omitted. - - -.. c:function:: void PyErr_BadInternalCall() - - This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, - where *message* indicates that an internal operation (e.g. a Python/C API - function) was invoked with an illegal argument. It is mostly for internal - use. - - -Issuing warnings -================ - -Use these functions to issue warnings from C code. They mirror similar -functions exported by the Python :mod:`warnings` module. They normally -print a warning message to *sys.stderr*; however, it is -also possible that the user has specified that warnings are to be turned into -errors, and in that case they will raise an exception. It is also possible that -the functions raise an exception because of a problem with the warning machinery. -The return value is ``0`` if no exception is raised, or ``-1`` if an exception -is raised. (It is not possible to determine whether a warning message is -actually printed, nor what the reason is for the exception; this is -intentional.) If an exception is raised, the caller should do its normal -exception handling (for example, :c:func:`Py_DECREF` owned references and return -an error value). - -.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) - - Issue a warning message. The *category* argument is a warning category (see - below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a - positive number giving a number of stack frames; the warning will be issued from - the currently executing line of code in that stack frame. A *stack_level* of 1 - is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, - and so forth. - - Warning categories must be subclasses of :c:data:`PyExc_Warning`; - :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; - the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard - Python warning categories are available as global variables whose names are - enumerated at :ref:`standardwarningcategories`. - - For information about warning control, see the documentation for the - :mod:`warnings` module and the :option:`-W` option in the command line - documentation. There is no C API for warning control. - -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) - - Much like :c:func:`PyErr_SetImportError` but this function allows for - specifying a subclass of :exc:`ImportError` to raise. - - .. versionadded:: 3.6 - - -.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) - - Issue a warning message with explicit control over all warning attributes. This - is a straightforward wrapper around the Python function - :func:`warnings.warn_explicit`, see there for more information. The *module* - and *registry* arguments may be set to ``NULL`` to get the default effect - described there. - - .. versionadded:: 3.4 - - -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) - - Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and - *module* are UTF-8 encoded strings, and *filename* is decoded from the - :term:`filesystem encoding and error handler`. - - -.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnEx`, but use - :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is - an ASCII-encoded string. - - .. versionadded:: 3.2 - - -.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. - - .. versionadded:: 3.6 - - -Querying the error indicator -============================ - -.. c:function:: PyObject* PyErr_Occurred() - - Test whether the error indicator is set. If set, return the exception *type* - (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not - own a reference to the return value, so you do not need to :c:func:`Py_DECREF` - it. - - The caller must hold the GIL. - - .. note:: - - Do not compare the return value to a specific exception; use - :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could - easily fail since the exception may be an instance instead of a class, in the - case of a class exception, or it may be a subclass of the expected exception.) - - -.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) - - Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This - should only be called when an exception is actually set; a memory access - violation will occur if no exception has been raised. - - -.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) - - Return true if the *given* exception matches the exception type in *exc*. If - *exc* is a class object, this also returns true when *given* is an instance - of a subclass. If *exc* is a tuple, all exception types in the tuple (and - recursively in subtuples) are searched for a match. - - -.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to ``NULL``. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be ``NULL`` even when the type object is not. - - .. note:: - - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: - - { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - /* ... code that might produce other errors ... */ - - PyErr_Restore(type, value, traceback); - } - - -.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are ``NULL``, the error - indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - -.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) - - Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below - can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance. - - .. note:: - - This function *does not* implicitly set the ``__traceback__`` - attribute on the exception value. If setting the traceback - appropriately is desired, the following additional snippet is needed:: - - if (tb != NULL) { - PyException_SetTraceback(val, tb); - } - - -.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. Returns new references for the three objects, any of which - may be ``NULL``. Does not modify the exception info state. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the - exception state. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) - - Set the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. - To clear the exception state, pass ``NULL`` for all three arguments. - For general rules about the three arguments, see :c:func:`PyErr_Restore`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception - state. - - .. versionadded:: 3.3 - - -Signal Handling -=============== - - -.. c:function:: int PyErr_CheckSignals() - - .. index:: - module: signal - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - This function interacts with Python's signal handling. It checks whether a - signal has been sent to the processes and if so, invokes the corresponding - signal handler. If the :mod:`signal` module is supported, this can invoke a - signal handler written in Python. In all cases, the default effect for - :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an - exception is raised the error indicator is set and the function returns ``-1``; - otherwise the function returns ``0``. The error indicator may or may not be - cleared if it was previously set. - - -.. c:function:: void PyErr_SetInterrupt() - - .. index:: - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - Simulate the effect of a :const:`SIGINT` signal arriving. The next time - :c:func:`PyErr_CheckSignals` is called, the Python signal handler for - :const:`SIGINT` will be called. - - If :const:`SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. - -.. c:function:: int PySignal_SetWakeupFd(int fd) - - This utility function specifies a file descriptor to which the signal number - is written as a single byte whenever a signal is received. *fd* must be - non-blocking. It returns the previous such file descriptor. - - The value ``-1`` disables the feature; this is the initial state. - This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any - error checking. *fd* should be a valid file descriptor. The function should - only be called from the main thread. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - -Exception Classes -================= - -.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) - - This utility function creates and returns a new exception class. The *name* - argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. - This creates a class object derived from :exc:`Exception` (accessible in C as - :c:data:`PyExc_Exception`). - - The :attr:`__module__` attribute of the new class is set to the first part (up - to the last dot) of the *name* argument, and the class name is set to the last - part (after the last dot). The *base* argument can be used to specify alternate - base classes; it can either be only one class or a tuple of classes. The *dict* - argument can be used to specify a dictionary of class variables and methods. - - -.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) - - Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-``NULL``, it will be used as the - docstring for the exception class. - - .. versionadded:: 3.2 - - -Exception Objects -================= - -.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) - - Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns ``NULL``. - - -.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) - - Set the traceback associated with the exception to *tb*. Use ``Py_None`` to - clear it. - - -.. c:function:: PyObject* PyException_GetContext(PyObject *ex) - - Return the context (another exception instance during whose handling *ex* was - raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. - - -.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - - Set the context associated with the exception to *ctx*. Use ``NULL`` to clear - it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. - - -.. c:function:: PyObject* PyException_GetCause(PyObject *ex) - - Return the cause (either an exception instance, or :const:`None`, - set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. - - -.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - - Set the cause associated with the exception to *cause*. Use ``NULL`` to clear - it. There is no type check to make sure that *cause* is either an exception - instance or :const:`None`. This steals a reference to *cause*. - - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. - - -.. _unicodeexceptions: - -Unicode Exception Objects -========================= - -The following functions are used to create and modify Unicode exceptions from C. - -.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - -.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - - .. deprecated:: 3.3 3.11 - - ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to - ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. - -.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeTranslateError` object with the attributes *object*, - *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. - - .. deprecated:: 3.3 3.11 - - ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to - ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) - - Return the *encoding* attribute of the given exception object. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) - - Return the *object* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) - - Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. - -.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) - - Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) - - Set the *end* attribute of the given exception object to *end*. Return ``0`` - on success, ``-1`` on failure. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) - - Return the *reason* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) - - Set the *reason* attribute of the given exception object to *reason*. Return - ``0`` on success, ``-1`` on failure. - - -.. _recursion: - -Recursion Control -================= - -These two functions provide a way to perform safe recursive calls at the C -level, both in the core and in extension modules. They are needed if the -recursive code does not necessarily invoke Python code (which tracks its -recursion depth automatically). -They are also not needed for *tp_call* implementations -because the :ref:`call protocol ` takes care of recursion handling. - -.. c:function:: int Py_EnterRecursiveCall(const char *where) - - Marks a point where a recursive C-level call is about to be performed. - - If :const:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the - case, a :exc:`RecursionError` is set and a nonzero value is returned. - Otherwise, zero is returned. - - *where* should be a UTF-8 encoded string such as ``" in instance check"`` to - be concatenated to the :exc:`RecursionError` message caused by the recursion - depth limit. - - .. versionchanged:: 3.9 - This function is now also available in the limited API. - -.. c:function:: void Py_LeaveRecursiveCall(void) - - Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each - *successful* invocation of :c:func:`Py_EnterRecursiveCall`. - - .. versionchanged:: 3.9 - This function is now also available in the limited API. - -Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires -special recursion handling. In addition to protecting the stack, -:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The -following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. - -.. c:function:: int Py_ReprEnter(PyObject *object) - - Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation - should return a string object indicating a cycle. As examples, - :class:`dict` objects return ``{...}`` and :class:`list` objects - return ``[...]``. - - The function will return a negative integer if the recursion limit - is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should - typically return ``NULL``. - - Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` - implementation can continue normally. - -.. c:function:: void Py_ReprLeave(PyObject *object) - - Ends a :c:func:`Py_ReprEnter`. Must be called once for each - invocation of :c:func:`Py_ReprEnter` that returns zero. - - -.. _standardexceptions: - -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.3 - :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, - :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, - :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, - :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, - :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, - :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, - :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` - and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. - -.. versionadded:: 3.5 - :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. - -.. versionadded:: 3.6 - :c:data:`PyExc_ModuleNotFoundError`. - -These are compatibility aliases to :c:data:`PyExc_OSError`: - -.. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | \(3) | -+-------------------------------------+----------+ - -.. versionchanged:: 3.3 - These aliases used to be separate exception types. - -Notes: - -(1) - This is a base class for other standard exceptions. - -(2) - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. - -.. _standardwarningcategories: - -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.2 - :c:data:`PyExc_ResourceWarning`. - -Notes: - -(1) - This is a base class for other standard warning categories. diff --git a/Doc/c-api/memory.rst.bak b/Doc/c-api/memory.rst.bak deleted file mode 100644 index 588668ee853c65a..000000000000000 --- a/Doc/c-api/memory.rst.bak +++ /dev/null @@ -1,642 +0,0 @@ -.. highlight:: c - - -.. _memory: - -***************** -Memory Management -***************** - -.. sectionauthor:: Vladimir Marangozov - - - -.. _memoryoverview: - -Overview -======== - -Memory management in Python involves a private heap containing all Python -objects and data structures. The management of this private heap is ensured -internally by the *Python memory manager*. The Python memory manager has -different components which deal with various dynamic storage management aspects, -like sharing, segmentation, preallocation or caching. - -At the lowest level, a raw memory allocator ensures that there is enough room in -the private heap for storing all Python-related data by interacting with the -memory manager of the operating system. On top of the raw memory allocator, -several object-specific allocators operate on the same heap and implement -distinct memory management policies adapted to the peculiarities of every object -type. For example, integer objects are managed differently within the heap than -strings, tuples or dictionaries because integers imply different storage -requirements and speed/space tradeoffs. The Python memory manager thus delegates -some of the work to the object-specific allocators, but ensures that the latter -operate within the bounds of the private heap. - -It is important to understand that the management of the Python heap is -performed by the interpreter itself and that the user has no control over it, -even if they regularly manipulate object pointers to memory blocks inside that -heap. The allocation of heap space for Python objects and other internal -buffers is performed on demand by the Python memory manager through the Python/C -API functions listed in this document. - -.. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() - -To avoid memory corruption, extension writers should never try to operate on -Python objects with the functions exported by the C library: :c:func:`malloc`, -:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed -calls between the C allocator and the Python memory manager with fatal -consequences, because they implement different algorithms and operate on -different heaps. However, one may safely allocate and release memory blocks -with the C library allocator for individual purposes, as shown in the following -example:: - - PyObject *res; - char *buf = (char *) malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - ...Do some I/O operation involving buf... - res = PyBytes_FromString(buf); - free(buf); /* malloc'ed */ - return res; - -In this example, the memory request for the I/O buffer is handled by the C -library allocator. The Python memory manager is involved only in the allocation -of the bytes object returned as a result. - -In most situations, however, it is recommended to allocate memory from the -Python heap specifically because the latter is under control of the Python -memory manager. For example, this is required when the interpreter is extended -with new object types written in C. Another reason for using the Python heap is -the desire to *inform* the Python memory manager about the memory needs of the -extension module. Even when the requested memory is used exclusively for -internal, highly-specific purposes, delegating all memory requests to the Python -memory manager causes the interpreter to have a more accurate image of its -memory footprint as a whole. Consequently, under certain circumstances, the -Python memory manager may or may not trigger appropriate actions, like garbage -collection, memory compaction or other preventive procedures. Note that by using -the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. - -.. seealso:: - - The :envvar:`PYTHONMALLOC` environment variable can be used to configure - the memory allocators used by Python. - - The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - statistics of the :ref:`pymalloc memory allocator ` every time a - new pymalloc object arena is created, and on shutdown. - -Allocator Domains -================= - -All allocating functions belong to one of three different "domains" (see also -:c:type:`PyMemAllocatorDomain`). These domains represent different allocation -strategies and are optimized for different purposes. The specific details on -how every domain allocates memory or what internal functions each domain calls -is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. - -The three allocation domains are: - -* Raw domain: intended for allocating memory for general-purpose memory - buffers where the allocation *must* go to the system allocator or where the - allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. - -* "Mem" domain: intended for allocating memory for Python buffers and - general-purpose memory buffers where the allocation must be performed with - the :term:`GIL` held. The memory is taken from the Python private heap. - -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. - -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. - -Raw Memory Interface -==================== - -The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. - -The :ref:`default raw memory allocator ` uses -the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` -and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting -zero bytes. - -.. versionadded:: 3.4 - -.. c:function:: void* PyMem_RawMalloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been - called instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will - be unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if - *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. - - If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* - remains a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_RawFree(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been - called before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _memoryinterface: - -Memory Interface -================ - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. versionchanged:: 3.6 - - The default allocator is now pymalloc instead of system :c:func:`malloc`. - -.. c:function:: void* PyMem_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - - If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or - :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - -The following type-oriented macros are provided for convenience. Note that -*TYPE* refers to any C type. - - -.. c:function:: TYPE* PyMem_New(TYPE, size_t n) - - Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have - been initialized in any way. - - -.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) - - Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, - *p* will be a pointer to the new memory area, or ``NULL`` in the event of - failure. - - This is a C preprocessor macro; *p* is always reassigned. Save the original - value of *p* to avoid losing memory when handling errors. - - -.. c:function:: void PyMem_Del(void *p) - - Same as :c:func:`PyMem_Free`. - -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. - -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` - - -Object allocators -================= - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -.. note:: - There is no guarantee that the memory returned by these allocators can be - succesfully casted to a Python object when intercepting the allocating - functions in this domain by the methods described in - the :ref:`Customize Memory Allocators ` section. - -The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. c:function:: void* PyObject_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyObject_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyObject_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - - If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyObject_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or - :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _default-memory-allocators: - -Default Memory Allocators -========================= - -Default memory allocators: - -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== - -Legend: - -* Name: value for :envvar:`PYTHONMALLOC` environment variable -* ``malloc``: system allocators from the standard C library, C functions: - :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` -* ``pymalloc``: :ref:`pymalloc memory allocator ` -* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` - -.. _customize-memory-allocators: - -Customize Memory Allocators -=========================== - -.. versionadded:: 3.4 - -.. c:type:: PyMemAllocatorEx - - Structure used to describe a memory block allocator. The structure has - four fields: - - +----------------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==========================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +----------------------------------------------------------+---------------------------------------+ - | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | - | | with zeros | - +----------------------------------------------------------+---------------------------------------+ - | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr)`` | free a memory block | - +----------------------------------------------------------+---------------------------------------+ - - .. versionchanged:: 3.5 - The :c:type:`PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - - -.. c:type:: PyMemAllocatorDomain - - Enum used to identify an allocator domain. Domains: - - .. c:macro:: PYMEM_DOMAIN_RAW - - Functions: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - - .. c:macro:: PYMEM_DOMAIN_MEM - - Functions: - - * :c:func:`PyMem_Malloc`, - * :c:func:`PyMem_Realloc` - * :c:func:`PyMem_Calloc` - * :c:func:`PyMem_Free` - - .. c:macro:: PYMEM_DOMAIN_OBJ - - Functions: - - * :c:func:`PyObject_Malloc` - * :c:func:`PyObject_Realloc` - * :c:func:`PyObject_Calloc` - * :c:func:`PyObject_Free` - -.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Get the memory block allocator of the specified domain. - - -.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-``NULL`` pointer when requesting - zero bytes. - - For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. - - If the new allocator is not a hook (does not call the previous allocator), - the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the - debug hooks on top on the new allocator. - - -.. c:function:: void PyMem_SetupDebugHooks(void) - - Setup hooks to detect bugs in the Python memory allocator functions. - - Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), - freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks - are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). - - Runtime checks: - - - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer - allocated by :c:func:`PyMem_Malloc` - - Detect write before the start of the buffer (buffer underflow) - - Detect write after the end of the buffer (buffer overflow) - - Check that the :term:`GIL ` is held when - allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: - :c:func:`PyMem_Malloc`) domains are called - - On error, the debug hooks use the :mod:`tracemalloc` module to get the - traceback where a memory block was allocated. The traceback is only - displayed if :mod:`tracemalloc` is tracing Python memory allocations and the - memory block was traced. - - These hooks are :ref:`installed by default ` if - Python is compiled in debug - mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install - debug hooks on a Python compiled in release mode. - - .. versionchanged:: 3.6 - This function now also works on Python compiled in release mode. - On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. The debug hooks now also check - if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and - :c:data:`PYMEM_DOMAIN_MEM` domains are called. - - .. versionchanged:: 3.8 - Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and - ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` - and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` - and ``free()``. - - -.. _pymalloc: - -The pymalloc allocator -====================== - -Python has a *pymalloc* allocator optimized for small objects (smaller or equal -to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and -:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. - -*pymalloc* is the :ref:`default allocator ` of the -:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and -:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. - -The arena allocator uses the following functions: - -* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, -* :c:func:`mmap` and :c:func:`munmap` if available, -* :c:func:`malloc` and :c:func:`free` otherwise. - -Customize pymalloc Arena Allocator ----------------------------------- - -.. versionadded:: 3.4 - -.. c:type:: PyObjectArenaAllocator - - Structure used to describe an arena allocator. The structure has - three fields: - - +--------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +--------------------------------------------------+---------------------------------------+ - | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | - +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | - +--------------------------------------------------+---------------------------------------+ - -.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) - - Get the arena allocator. - -.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) - - Set the arena allocator. - - -tracemalloc C API -================= - -.. versionadded:: 3.7 - -.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) - - Track an allocated memory block in the :mod:`tracemalloc` module. - - Return ``0`` on success, return ``-1`` on error (failed to allocate memory to - store the trace). Return ``-2`` if tracemalloc is disabled. - - If memory block is already tracked, update the existing trace. - -.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) - - Untrack an allocated memory block in the :mod:`tracemalloc` module. - Do nothing if the block was not tracked. - - Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. - - -.. _memoryexamples: - -Examples -======== - -Here is the example from section :ref:`memoryoverview`, rewritten so that the -I/O buffer is allocated from the Python heap by using the first function set:: - - PyObject *res; - char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Free(buf); /* allocated with PyMem_Malloc */ - return res; - -The same code using the type-oriented function set:: - - PyObject *res; - char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ - return res; - -Note that in the two examples above, the buffer is always manipulated via -functions belonging to the same set. Indeed, it is required to use the same -memory API family for a given memory block, so that the risk of mixing different -allocators is reduced to a minimum. The following code sequence contains two -errors, one of which is labeled as *fatal* because it mixes two different -allocators operating on different heaps. :: - - char *buf1 = PyMem_New(char, BUFSIZ); - char *buf2 = (char *) malloc(BUFSIZ); - char *buf3 = (char *) PyMem_Malloc(BUFSIZ); - ... - PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ - free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ - -In addition to the functions aimed at handling raw memory blocks from the Python -heap, objects in Python are allocated and released with :c:func:`PyObject_New`, -:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. - -These will be explained in the next chapter on defining and implementing new -object types in C. diff --git a/Doc/faq/design.rst.bak b/Doc/faq/design.rst.bak deleted file mode 100644 index 7fe1c6d58f58a12..000000000000000 --- a/Doc/faq/design.rst.bak +++ /dev/null @@ -1,759 +0,0 @@ -====================== -Design and History FAQ -====================== - -.. only:: html - - .. contents:: - - -Why does Python use indentation for grouping of statements? ------------------------------------------------------------ - -Guido van Rossum believes that using indentation for grouping is extremely -elegant and contributes a lot to the clarity of the average Python program. -Most people learn to love this feature after a while. - -Since there are no begin/end brackets there cannot be a disagreement between -grouping perceived by the parser and the human reader. Occasionally C -programmers will encounter a fragment of code like this:: - - if (x <= y) - x++; - y--; - z++; - -Only the ``x++`` statement is executed if the condition is true, but the -indentation leads many to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering as to why ``y`` is being decremented even -for ``x > y``. - -Because there are no begin/end brackets, Python is much less prone to -coding-style conflicts. In C there are many different ways to place the braces. -After becoming used to reading and writing code using a particular style, -it is normal to feel somewhat uneasy when reading (or being required to write) -in a different one. - - -Many coding styles place begin/end brackets on a line by themselves. This makes -programs considerably longer and wastes valuable screen space, making it harder -to get a good overview of a program. Ideally, a function should fit on one -screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 -lines of C. This is not solely due to the lack of begin/end brackets -- the -lack of declarations and the high-level data types are also responsible -- but -the indentation-based syntax certainly helps. - - -Why am I getting strange results with simple arithmetic operations? -------------------------------------------------------------------- - -See the next question. - - -Why are floating-point calculations so inaccurate? --------------------------------------------------- - -Users are often surprised by results like this:: - - >>> 1.2 - 1.0 - 0.19999999999999996 - -and think it is a bug in Python. It's not. This has little to do with Python, -and much more to do with how the underlying platform handles floating-point -numbers. - -The :class:`float` type in CPython uses a C ``double`` for storage. A -:class:`float` object's value is stored in binary floating-point with a fixed -precision (typically 53 bits) and Python uses C operations, which in turn rely -on the hardware implementation in the processor, to perform floating-point -operations. This means that as far as floating-point operations are concerned, -Python behaves like many popular languages including C and Java. - -Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: - - >>> x = 1.2 - -the value stored for ``x`` is a (very good) approximation to the decimal value -``1.2``, but is not exactly equal to it. On a typical machine, the actual -stored value is:: - - 1.0011001100110011001100110011001100110011001100110011 (binary) - -which is exactly:: - - 1.1999999999999999555910790149937383830547332763671875 (decimal) - -The typical precision of 53 bits provides Python floats with 15--16 -decimal digits of accuracy. - -For a fuller explanation, please see the :ref:`floating point arithmetic -` chapter in the Python tutorial. - - -Why are Python strings immutable? ---------------------------------- - -There are several advantages. - -One is performance: knowing that a string is immutable means we can allocate -space for it at creation time, and the storage requirements are fixed and -unchanging. This is also one of the reasons for the distinction between tuples -and lists. - -Another advantage is that strings in Python are considered as "elemental" as -numbers. No amount of activity will change the value 8 to anything else, and in -Python, no amount of activity will change the string "eight" to anything else. - - -.. _why-self: - -Why must 'self' be used explicitly in method definitions and calls? -------------------------------------------------------------------- - -The idea was borrowed from Modula-3. It turns out to be very useful, for a -variety of reasons. - -First, it's more obvious that you are using a method or instance attribute -instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it -absolutely clear that an instance variable or method is used even if you don't -know the class definition by heart. In C++, you can sort of tell by the lack of -a local variable declaration (assuming globals are rare or easily recognizable) --- but in Python, there are no local variable declarations, so you'd have to -look up the class definition to be sure. Some C++ and Java coding standards -call for instance attributes to have an ``m_`` prefix, so this explicitness is -still useful in those languages, too. - -Second, it means that no special syntax is necessary if you want to explicitly -reference or call the method from a particular class. In C++, if you want to -use a method from a base class which is overridden in a derived class, you have -to use the ``::`` operator -- in Python you can write -``baseclass.methodname(self, )``. This is particularly useful -for :meth:`__init__` methods, and in general in cases where a derived class -method wants to extend the base class method of the same name and thus has to -call the base class method somehow. - -Finally, for instance variables it solves a syntactic problem with assignment: -since local variables in Python are (by definition!) those variables to which a -value is assigned in a function body (and that aren't explicitly declared -global), there has to be some way to tell the interpreter that an assignment was -meant to assign to an instance variable instead of to a local variable, and it -should preferably be syntactic (for efficiency reasons). C++ does this through -declarations, but Python doesn't have declarations and it would be a pity having -to introduce them just for this purpose. Using the explicit ``self.var`` solves -this nicely. Similarly, for using instance variables, having to write -``self.var`` means that references to unqualified names inside a method don't -have to search the instance's directories. To put it another way, local -variables and instance variables live in two different namespaces, and you need -to tell Python which namespace to use. - - -.. _why-can-t-i-use-an-assignment-in-an-expression: - -Why can't I use an assignment in an expression? ------------------------------------------------ - -Starting in Python 3.8, you can! - -Assignment expressions using the walrus operator `:=` assign a variable in an -expression:: - - while chunk := fp.read(200): - print(chunk) - -See :pep:`572` for more information. - - - -Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? ----------------------------------------------------------------------------------------------------------------- - -As Guido said: - - (a) For some operations, prefix notation just reads better than - postfix -- prefix (and infix!) operations have a long tradition in - mathematics which likes notations where the visuals help the - mathematician thinking about a problem. Compare the easy with which we - rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of - doing the same thing using a raw OO notation. - - (b) When I read code that says len(x) I *know* that it is asking for - the length of something. This tells me two things: the result is an - integer, and the argument is some kind of container. To the contrary, - when I read x.len(), I have to already know that x is some kind of - container implementing an interface or inheriting from a class that - has a standard len(). Witness the confusion we occasionally have when - a class that is not implementing a mapping has a get() or keys() - method, or something that isn't a file has a write() method. - - -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html - - -Why is join() a string method instead of a list or tuple method? ----------------------------------------------------------------- - -Strings became much more like other standard types starting in Python 1.6, when -methods were added which give the same functionality that has always been -available using the functions of the string module. Most of these new methods -have been widely accepted, but the one which appears to make some programmers -feel uncomfortable is:: - - ", ".join(['1', '2', '4', '8', '16']) - -which gives the result:: - - "1, 2, 4, 8, 16" - -There are two common arguments against this usage. - -The first runs along the lines of: "It looks really ugly using a method of a -string literal (string constant)", to which the answer is that it might, but a -string literal is just a fixed value. If the methods are to be allowed on names -bound to strings there is no logical reason to make them unavailable on -literals. - -The second objection is typically cast as: "I am really telling a sequence to -join its members together with a string constant". Sadly, you aren't. For some -reason there seems to be much less difficulty with having :meth:`~str.split` as -a string method, since in that case it is easy to see that :: - - "1, 2, 4, 8, 16".split(", ") - -is an instruction to a string literal to return the substrings delimited by the -given separator (or, by default, arbitrary runs of white space). - -:meth:`~str.join` is a string method because in using it you are telling the -separator string to iterate over a sequence of strings and insert itself between -adjacent elements. This method can be used with any argument which obeys the -rules for sequence objects, including any new classes you might define yourself. -Similar methods exist for bytes and bytearray objects. - - -How fast are exceptions? ------------------------- - -A try/except block is extremely efficient if no exceptions are raised. Actually -catching an exception is expensive. In versions of Python prior to 2.0 it was -common to use this idiom:: - - try: - value = mydict[key] - except KeyError: - mydict[key] = getvalue(key) - value = mydict[key] - -This only made sense when you expected the dict to have the key almost all the -time. If that wasn't the case, you coded it like this:: - - if key in mydict: - value = mydict[key] - else: - value = mydict[key] = getvalue(key) - -For this specific case, you could also use ``value = dict.setdefault(key, -getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it -is evaluated in all cases. - - -Why isn't there a switch or case statement in Python? ------------------------------------------------------ - -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. - -For cases where you need to choose from a very large number of possibilities, -you can create a dictionary mapping case values to functions to call. For -example:: - - def function_1(...): - ... - - functions = {'a': function_1, - 'b': function_2, - 'c': self.method_1, ...} - - func = functions[value] - func() - -For calling methods on objects, you can simplify yet further by using the -:func:`getattr` built-in to retrieve methods with a particular name:: - - def visit_a(self, ...): - ... - ... - - def dispatch(self, value): - method_name = 'visit_' + str(value) - method = getattr(self, method_name) - method() - -It's suggested that you use a prefix for the method names, such as ``visit_`` in -this example. Without such a prefix, if values are coming from an untrusted -source, an attacker would be able to call any method on your object. - - -Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? --------------------------------------------------------------------------------------------------------- - -Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for -each Python stack frame. Also, extensions can call back into Python at almost -random moments. Therefore, a complete threads implementation requires thread -support for C. - -Answer 2: Fortunately, there is `Stackless Python `_, -which has a completely redesigned interpreter loop that avoids the C stack. - - -Why can't lambda expressions contain statements? ------------------------------------------------- - -Python lambda expressions cannot contain statements because Python's syntactic -framework can't handle statements nested inside expressions. However, in -Python, this is not a serious problem. Unlike lambda forms in other languages, -where they add functionality, Python lambdas are only a shorthand notation if -you're too lazy to define a function. - -Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda instead of a -locally-defined function is that you don't need to invent a name for the -function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda expression yields) is assigned! - - -Can Python be compiled to machine code, C or some other language? ------------------------------------------------------------------ - -`Cython `_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka `_ is -an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC `_. - - -How does Python manage memory? ------------------------------- - -The details of Python memory management depend on the implementation. The -standard implementation of Python, :term:`CPython`, uses reference counting to -detect inaccessible objects, and another mechanism to collect reference cycles, -periodically executing a cycle detection algorithm which looks for inaccessible -cycles and deletes the objects involved. The :mod:`gc` module provides functions -to perform a garbage collection, obtain debugging statistics, and tune the -collector's parameters. - -Other implementations (such as `Jython `_ or -`PyPy `_), however, can rely on a different mechanism -such as a full-blown garbage collector. This difference can cause some -subtle porting problems if your Python code depends on the behavior of the -reference counting implementation. - -In some Python implementations, the following code (which is fine in CPython) -will probably run out of file descriptors:: - - for file in very_long_list_of_files: - f = open(file) - c = f.read(1) - -Indeed, using CPython's reference counting and destructor scheme, each new -assignment to *f* closes the previous file. With a traditional GC, however, -those file objects will only get collected (and closed) at varying and possibly -long intervals. - -If you want to write code that will work with any Python implementation, -you should explicitly close the file or use the :keyword:`with` statement; -this will work regardless of memory management scheme:: - - for file in very_long_list_of_files: - with open(file) as f: - c = f.read(1) - - -Why doesn't CPython use a more traditional garbage collection scheme? ---------------------------------------------------------------------- - -For one thing, this is not a C standard feature and hence it's not portable. -(Yes, we know about the Boehm GC library. It has bits of assembler code for -*most* common platforms, not for all of them, and although it is mostly -transparent, it isn't completely transparent; patches are required to get -Python to work with it.) - -Traditional GC also becomes a problem when Python is embedded into other -applications. While in a standalone Python it's fine to replace the standard -malloc() and free() with versions provided by the GC library, an application -embedding Python may want to have its *own* substitute for malloc() and free(), -and may not want Python's. Right now, CPython works with anything that -implements malloc() and free() properly. - - -Why isn't all memory freed when CPython exits? ----------------------------------------------- - -Objects referenced from the global namespaces of Python modules are not always -deallocated when Python exits. This may happen if there are circular -references. There are also certain bits of memory that are allocated by the C -library that are impossible to free (e.g. a tool like Purify will complain about -these). Python is, however, aggressive about cleaning up memory on exit and -does try to destroy every single object. - -If you want to force Python to delete certain things on deallocation use the -:mod:`atexit` module to run a function that will force those deletions. - - -Why are there separate tuple and list data types? -------------------------------------------------- - -Lists and tuples, while similar in many respects, are generally used in -fundamentally different ways. Tuples can be thought of as being similar to -Pascal records or C structs; they're small collections of related data which may -be of different types which are operated on as a group. For example, a -Cartesian coordinate is appropriately represented as a tuple of two or three -numbers. - -Lists, on the other hand, are more like arrays in other languages. They tend to -hold a varying number of objects all of which have the same type and which are -operated on one-by-one. For example, ``os.listdir('.')`` returns a list of -strings representing the files in the current directory. Functions which -operate on this output would generally not break if you added another file or -two to the directory. - -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. - - -How are lists implemented in CPython? -------------------------------------- - -CPython's lists are really variable-length arrays, not Lisp-style linked lists. -The implementation uses a contiguous array of references to other objects, and -keeps a pointer to this array and the array's length in a list head structure. - -This makes indexing a list ``a[i]`` an operation whose cost is independent of -the size of the list or the value of the index. - -When items are appended or inserted, the array of references is resized. Some -cleverness is applied to improve the performance of appending items repeatedly; -when the array must be grown, some extra space is allocated so the next few -times don't require an actual resize. - - -How are dictionaries implemented in CPython? --------------------------------------------- - -CPython's dictionaries are implemented as resizable hash tables. Compared to -B-trees, this gives better performance for lookup (the most common operation by -far) under most circumstances, and the implementation is simpler. - -Dictionaries work by computing a hash code for each key stored in the dictionary -using the :func:`hash` built-in function. The hash code varies widely depending -on the key and a per-process seed; for example, "Python" could hash to --539294296 while "python", a string that differs by a single bit, could hash -to 1142331976. The hash code is then used to calculate a location in an -internal array where the value will be stored. Assuming that you're storing -keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. - - -Why must dictionary keys be immutable? --------------------------------------- - -The hash table implementation of dictionaries uses a hash value calculated from -the key value to find the key. If the key were a mutable object, its value -could change, and thus its hash could also change. But since whoever changes -the key object can't tell that it was being used as a dictionary key, it can't -move the entry around in the dictionary. Then, when you try to look up the same -object in the dictionary it won't be found because its hash value is different. -If you tried to look up the old value it wouldn't be found either, because the -value of the object found in that hash bin would be different. - -If you want a dictionary indexed with a list, simply convert the list to a tuple -first; the function ``tuple(L)`` creates a tuple with the same entries as the -list ``L``. Tuples are immutable and can therefore be used as dictionary keys. - -Some unacceptable solutions that have been proposed: - -- Hash lists by their address (object ID). This doesn't work because if you - construct a new list with the same value it won't be found; e.g.:: - - mydict = {[1, 2]: '12'} - print(mydict[[1, 2]]) - - would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the - second line differs from that in the first line. In other words, dictionary - keys should be compared using ``==``, not using :keyword:`is`. - -- Make a copy when using a list as a key. This doesn't work because the list, - being a mutable object, could contain a reference to itself, and then the - copying code would run into an infinite loop. - -- Allow lists as keys but tell the user not to modify them. This would allow a - class of hard-to-track bugs in programs when you forgot or modified a list by - accident. It also invalidates an important invariant of dictionaries: every - value in ``d.keys()`` is usable as a key of the dictionary. - -- Mark lists as read-only once they are used as a dictionary key. The problem - is that it's not just the top-level object that could change its value; you - could use a tuple containing a list as a key. Entering anything as a key into - a dictionary would require marking all objects reachable from there as - read-only -- and again, self-referential objects could cause an infinite loop. - -There is a trick to get around this if you need to, but use it at your own risk: -You can wrap a mutable structure inside a class instance which has both a -:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the -hash value for all such wrapper objects that reside in a dictionary (or other -hash based structure), remain fixed while the object is in the dictionary (or -other structure). :: - - class ListWrapper: - def __init__(self, the_list): - self.the_list = the_list - - def __eq__(self, other): - return self.the_list == other.the_list - - def __hash__(self): - l = self.the_list - result = 98767 - len(l)*555 - for i, el in enumerate(l): - try: - result = result + (hash(el) % 9999999) * 1001 + i - except Exception: - result = (result % 7777777) + i * 333 - return result - -Note that the hash computation is complicated by the possibility that some -members of the list may be unhashable and also by the possibility of arithmetic -overflow. - -Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) -is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), -regardless of whether the object is in a dictionary or not. If you fail to meet -these restrictions dictionaries and other hash based structures will misbehave. - -In the case of ListWrapper, whenever the wrapper object is in a dictionary the -wrapped list must not change to avoid anomalies. Don't do this unless you are -prepared to think hard about the requirements and the consequences of not -meeting them correctly. Consider yourself warned. - - -Why doesn't list.sort() return the sorted list? ------------------------------------------------ - -In situations where performance matters, making a copy of the list just to sort -it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In -order to remind you of that fact, it does not return the sorted list. This way, -you won't be fooled into accidentally overwriting a list when you need a sorted -copy but also need to keep the unsorted version around. - -If you want to return a new list, use the built-in :func:`sorted` function -instead. This function creates a new list from a provided iterable, sorts -it and returns it. For example, here's how to iterate over the keys of a -dictionary in sorted order:: - - for key in sorted(mydict): - ... # do whatever with mydict[key]... - - -How do you specify and enforce an interface spec in Python? ------------------------------------------------------------ - -An interface specification for a module as provided by languages such as C++ and -Java describes the prototypes for the methods and functions of the module. Many -feel that compile-time enforcement of interface specifications helps in the -construction of large programs. - -Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes -(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check -whether an instance or a class implements a particular ABC. The -:mod:`collections.abc` module defines a set of useful ABCs such as -:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and -:class:`~collections.abc.MutableMapping`. - -For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. - -A good test suite for a module can both provide a regression test and serve as a -module interface specification and a set of examples. Many Python modules can -be run as a script to provide a simple "self test." Even modules which use -complex external interfaces can often be tested in isolation using trivial -"stub" emulations of the external interface. The :mod:`doctest` and -:mod:`unittest` modules or third-party test frameworks can be used to construct -exhaustive test suites that exercise every line of code in a module. - -An appropriate testing discipline can help build large complex applications in -Python as well as having interface specifications would. In fact, it can be -better because an interface specification cannot test certain properties of a -program. For example, the :meth:`append` method is expected to add new elements -to the end of some internal list; an interface specification cannot test that -your :meth:`append` implementation will actually do this correctly, but it's -trivial to check this property in a test suite. - -Writing test suites is very helpful, and you might want to design your code to -make it easily tested. One increasingly popular technique, test-driven -development, calls for writing parts of the test suite first, before you write -any of the actual code. Of course Python allows you to be sloppy and not write -test cases at all. - - -Why is there no goto? ---------------------- - -You can use exceptions to provide a "structured goto" that even works across -function calls. Many feel that exceptions can conveniently emulate all -reasonable uses of the "go" or "goto" constructs of C, Fortran, and other -languages. For example:: - - class label(Exception): pass # declare a label - - try: - ... - if condition: raise label() # goto label - ... - except label: # where to goto - pass - ... - -This doesn't allow you to jump into the middle of a loop, but that's usually -considered an abuse of goto anyway. Use sparingly. - - -Why can't raw strings (r-strings) end with a backslash? -------------------------------------------------------- - -More precisely, they can't end with an odd number of backslashes: the unpaired -backslash at the end escapes the closing quote character, leaving an -unterminated string. - -Raw strings were designed to ease creating input for processors (chiefly regular -expression engines) that want to do their own backslash escape processing. Such -processors consider an unmatched trailing backslash to be an error anyway, so -raw strings disallow that. In return, they allow you to pass on the string -quote character by escaping it with a backslash. These rules work well when -r-strings are used for their intended purpose. - -If you're trying to build Windows pathnames, note that all Windows system calls -accept forward slashes too:: - - f = open("/mydir/file.txt") # works fine! - -If you're trying to build a pathname for a DOS command, try e.g. one of :: - - dir = r"\this\is\my\dos\dir" "\\" - dir = r"\this\is\my\dos\dir\ "[:-1] - dir = "\\this\\is\\my\\dos\\dir\\" - - -Why doesn't Python have a "with" statement for attribute assignments? ---------------------------------------------------------------------- - -Python has a 'with' statement that wraps the execution of a block, calling code -on the entrance and exit from the block. Some languages have a construct that -looks like this:: - - with obj: - a = 1 # equivalent to obj.a = 1 - total = total + 1 # obj.total = obj.total + 1 - -In Python, such a construct would be ambiguous. - -Other languages, such as Object Pascal, Delphi, and C++, use static types, so -it's possible to know, in an unambiguous way, what member is being assigned -to. This is the main point of static typing -- the compiler *always* knows the -scope of every variable at compile time. - -Python uses dynamic types. It is impossible to know in advance which attribute -will be referenced at runtime. Member attributes may be added or removed from -objects on the fly. This makes it impossible to know, from a simple reading, -what attribute is being referenced: a local one, a global one, or a member -attribute? - -For instance, take the following incomplete snippet:: - - def foo(a): - with a: - print(x) - -The snippet assumes that "a" must have a member attribute called "x". However, -there is nothing in Python that tells the interpreter this. What should happen -if "a" is, let us say, an integer? If there is a global variable named "x", -will it be used inside the with block? As you see, the dynamic nature of Python -makes such choices much harder. - -The primary benefit of "with" and similar language features (reduction of code -volume) can, however, easily be achieved in Python by assignment. Instead of:: - - function(args).mydict[index][index].a = 21 - function(args).mydict[index][index].b = 42 - function(args).mydict[index][index].c = 63 - -write this:: - - ref = function(args).mydict[index][index] - ref.a = 21 - ref.b = 42 - ref.c = 63 - -This also has the side-effect of increasing execution speed because name -bindings are resolved at run-time in Python, and the second version only needs -to perform the resolution once. - - -Why are colons required for the if/while/def/class statements? --------------------------------------------------------------- - -The colon is required primarily to enhance readability (one of the results of -the experimental ABC language). Consider this:: - - if a == b - print(a) - -versus :: - - if a == b: - print(a) - -Notice how the second one is slightly easier to read. Notice further how a -colon sets off the example in this FAQ answer; it's a standard usage in English. - -Another minor reason is that the colon makes it easier for editors with syntax -highlighting; they can look for colons to decide when indentation needs to be -increased instead of having to do a more elaborate parsing of the program text. - - -Why does Python allow commas at the end of lists and tuples? ------------------------------------------------------------- - -Python lets you add a trailing comma at the end of lists, tuples, and -dictionaries:: - - [1, 2, 3,] - ('a', 'b', 'c',) - d = { - "A": [1, 5], - "B": [6, 7], # last trailing comma is optional but good style - } - - -There are several reasons to allow this. - -When you have a literal value for a list, tuple, or dictionary spread across -multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be reordered -without creating a syntax error. - -Accidentally omitting the comma can lead to errors that are hard to diagnose. -For example:: - - x = [ - "fee", - "fie" - "foo", - "fum" - ] - -This list looks like it has four elements, but it actually contains three: -"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. - -Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/howto/descriptor.rst.bak b/Doc/howto/descriptor.rst.bak deleted file mode 100644 index 94a8b4e6b40b962..000000000000000 --- a/Doc/howto/descriptor.rst.bak +++ /dev/null @@ -1,1575 +0,0 @@ -.. _descriptorhowto: - -====================== -Descriptor HowTo Guide -====================== - -:Author: Raymond Hettinger -:Contact: - -.. Contents:: - - -:term:`Descriptors ` let objects customize attribute lookup, -storage, and deletion. - -This guide has four major sections: - -1) The "primer" gives a basic overview, moving gently from simple examples, - adding one feature at a time. Start here if you're new to descriptors. - -2) The second section shows a complete, practical descriptor example. If you - already know the basics, start there. - -3) The third section provides a more technical tutorial that goes into the - detailed mechanics of how descriptors work. Most people don't need this - level of detail. - -4) The last section has pure Python equivalents for built-in descriptors that - are written in C. Read this if you're curious about how functions turn - into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and - :term:`__slots__`. - - -Primer -^^^^^^ - -In this primer, we start with the most basic possible example and then we'll -add new capabilities one by one. - - -Simple example: A descriptor that returns a constant ----------------------------------------------------- - -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always -returns the constant ``10``: - -.. testcode:: - - class Ten: - def __get__(self, obj, objtype=None): - return 10 - -To use the descriptor, it must be stored as a class variable in another class: - -.. testcode:: - - class A: - x = 5 # Regular class attribute - y = Ten() # Descriptor instance - -An interactive session shows the difference between normal attribute lookup -and descriptor lookup: - -.. doctest:: - - >>> a = A() # Make an instance of class A - >>> a.x # Normal attribute lookup - 5 - >>> a.y # Descriptor lookup - 10 - -In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` -in the class dictionary. In the ``a.y`` lookup, the dot operator -finds a descriptor instance, recognized by its ``__get__`` method. -Calling that method returns ``10``. - -Note that the value ``10`` is not stored in either the class dictionary or the -instance dictionary. Instead, the value ``10`` is computed on demand. - -This example shows how a simple descriptor works, but it isn't very useful. -For retrieving constants, normal attribute lookup would be better. - -In the next section, we'll create something more useful, a dynamic lookup. - - -Dynamic lookups ---------------- - -Interesting descriptors typically run computations instead of returning -constants: - -.. testcode:: - - import os - - class DirectorySize: - - def __get__(self, obj, objtype=None): - return len(os.listdir(obj.dirname)) - - class Directory: - - size = DirectorySize() # Descriptor instance - - def __init__(self, dirname): - self.dirname = dirname # Regular instance attribute - -An interactive session shows that the lookup is dynamic — it computes -different, updated answers each time:: - - >>> s = Directory('songs') - >>> g = Directory('games') - >>> s.size # The songs directory has twenty files - 20 - >>> g.size # The games directory has three files - 3 - >>> open('games/newfile').close() # Add a fourth file to the directory - >>> g.size # File count is automatically updated - 4 - -Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* -parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is -either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* -parameter is the class *Directory*. - - -Managed attributes ------------------- - -A popular use for descriptors is managing access to instance data. The -descriptor is assigned to a public attribute in the class dictionary while the -actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when -the public attribute is accessed. - -In the following example, *age* is the public attribute and *_age* is the -private attribute. When the public attribute is accessed, the descriptor logs -the lookup or update: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAgeAccess: - - def __get__(self, obj, objtype=None): - value = obj._age - logging.info('Accessing %r giving %r', 'age', value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', 'age', value) - obj._age = value - - class Person: - - age = LoggedAgeAccess() # Descriptor instance - - def __init__(self, name, age): - self.name = name # Regular instance attribute - self.age = age # Calls __set__() - - def birthday(self): - self.age += 1 # Calls both __get__() and __set__() - - -An interactive session shows that all access to the managed attribute *age* is -logged, but that the regular attribute *name* is not logged: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> mary = Person('Mary M', 30) # The initial age update is logged - INFO:root:Updating 'age' to 30 - >>> dave = Person('David D', 40) - INFO:root:Updating 'age' to 40 - - >>> vars(mary) # The actual data is in a private attribute - {'name': 'Mary M', '_age': 30} - >>> vars(dave) - {'name': 'David D', '_age': 40} - - >>> mary.age # Access the data and log the lookup - INFO:root:Accessing 'age' giving 30 - 30 - >>> mary.birthday() # Updates are logged as well - INFO:root:Accessing 'age' giving 30 - INFO:root:Updating 'age' to 31 - - >>> dave.name # Regular attribute lookup isn't logged - 'David D' - >>> dave.age # Only the managed attribute is logged - INFO:root:Accessing 'age' giving 40 - 40 - -One major issue with this example is that the private name *_age* is hardwired in -the *LoggedAgeAccess* class. That means that each instance can only have one -logged attribute and that its name is unchangeable. In the next example, -we'll fix that problem. - - -Customized names ----------------- - -When a class uses descriptors, it can inform each descriptor about which -variable name was used. - -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can -be recorded, giving each descriptor its own *public_name* and *private_name*: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAccess: - - def __set_name__(self, owner, name): - self.public_name = name - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - value = getattr(obj, self.private_name) - logging.info('Accessing %r giving %r', self.public_name, value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', self.public_name, value) - setattr(obj, self.private_name, value) - - class Person: - - name = LoggedAccess() # First descriptor instance - age = LoggedAccess() # Second descriptor instance - - def __init__(self, name, age): - self.name = name # Calls the first descriptor - self.age = age # Calls the second descriptor - - def birthday(self): - self.age += 1 - -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here -we call :func:`vars` to look up the descriptor without triggering it: - -.. doctest:: - - >>> vars(vars(Person)['name']) - {'public_name': 'name', 'private_name': '_name'} - >>> vars(vars(Person)['age']) - {'public_name': 'age', 'private_name': '_age'} - -The new class now logs access to both *name* and *age*: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> pete = Person('Peter P', 10) - INFO:root:Updating 'name' to 'Peter P' - INFO:root:Updating 'age' to 10 - >>> kate = Person('Catherine C', 20) - INFO:root:Updating 'name' to 'Catherine C' - INFO:root:Updating 'age' to 20 - -The two *Person* instances contain only the private names:: - - >>> vars(pete) - {'_name': 'Peter P', '_age': 10} - >>> vars(kate) - {'_name': 'Catherine C', '_age': 20} - - -Closing thoughts ----------------- - -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. - -Optionally, descriptors can have a :meth:`__set_name__` method. This is only -used in cases where a descriptor needs to know either the class where it was -created or the name of class variable it was assigned to. (This method, if -present, is called even if the class is not a descriptor.) - -Descriptors get invoked by the dot operator during attribute lookup. If a -descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, -the descriptor instance is returned without invoking it. - -Descriptors only work when used as class variables. When put in instances, -they have no effect. - -The main motivation for descriptors is to provide a hook allowing objects -stored in class variables to control what happens during attribute lookup. - -Traditionally, the calling class controls what happens during lookup. -Descriptors invert that relationship and allow the data being looked-up to -have a say in the matter. - -Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as -descriptors. - - -Complete Practical Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this example, we create a practical and powerful tool for locating -notoriously hard to find data corruption bugs. - - -Validator class ---------------- - -A validator is a descriptor for managed attribute access. Prior to storing -any data, it verifies that the new value meets various type and range -restrictions. If those restrictions aren't met, it raises an exception to -prevent data corruption at its source. - -This :class:`Validator` class is both an :term:`abstract base class` and a -managed attribute descriptor: - -.. testcode:: - - from abc import ABC, abstractmethod - - class Validator(ABC): - - def __set_name__(self, owner, name): - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - return getattr(obj, self.private_name) - - def __set__(self, obj, value): - self.validate(value) - setattr(obj, self.private_name, value) - - @abstractmethod - def validate(self, value): - pass - -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. - - -Custom validators ------------------ - -Here are three practical data validation utilities: - -1) :class:`OneOf` verifies that a value is one of a restricted set of options. - -2) :class:`Number` verifies that a value is either an :class:`int` or - :class:`float`. Optionally, it verifies that a value is between a given - minimum or maximum. - -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it - validates a given minimum or maximum length. It can validate a - user-defined `predicate - `_ as well. - -.. testcode:: - - class OneOf(Validator): - - def __init__(self, *options): - self.options = set(options) - - def validate(self, value): - if value not in self.options: - raise ValueError(f'Expected {value!r} to be one of {self.options!r}') - - class Number(Validator): - - def __init__(self, minvalue=None, maxvalue=None): - self.minvalue = minvalue - self.maxvalue = maxvalue - - def validate(self, value): - if not isinstance(value, (int, float)): - raise TypeError(f'Expected {value!r} to be an int or float') - if self.minvalue is not None and value < self.minvalue: - raise ValueError( - f'Expected {value!r} to be at least {self.minvalue!r}' - ) - if self.maxvalue is not None and value > self.maxvalue: - raise ValueError( - f'Expected {value!r} to be no more than {self.maxvalue!r}' - ) - - class String(Validator): - - def __init__(self, minsize=None, maxsize=None, predicate=None): - self.minsize = minsize - self.maxsize = maxsize - self.predicate = predicate - - def validate(self, value): - if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') - if self.minsize is not None and len(value) < self.minsize: - raise ValueError( - f'Expected {value!r} to be no smaller than {self.minsize!r}' - ) - if self.maxsize is not None and len(value) > self.maxsize: - raise ValueError( - f'Expected {value!r} to be no bigger than {self.maxsize!r}' - ) - if self.predicate is not None and not self.predicate(value): - raise ValueError( - f'Expected {self.predicate} to be true for {value!r}' - ) - - -Practical application ---------------------- - -Here's how the data validators can be used in a real class: - -.. testcode:: - - class Component: - - name = String(minsize=3, maxsize=10, predicate=str.isupper) - kind = OneOf('wood', 'metal', 'plastic') - quantity = Number(minvalue=0) - - def __init__(self, name, kind, quantity): - self.name = name - self.kind = kind - self.quantity = quantity - -The descriptors prevent invalid instances from being created: - -.. doctest:: - - >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase - Traceback (most recent call last): - ... - ValueError: Expected to be true for 'Widget' - - >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled - Traceback (most recent call last): - ... - ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} - - >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative - Traceback (most recent call last): - ... - ValueError: Expected -5 to be at least 0 - >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number - Traceback (most recent call last): - ... - TypeError: Expected 'V' to be an int or float - - >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid - - -Technical Tutorial -^^^^^^^^^^^^^^^^^^ - -What follows is a more technical tutorial for the mechanics and details of how -descriptors work. - - -Abstract --------- - -Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Provides an example showing how object relational mappings work. - -Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works. - - -Definition and introduction ---------------------------- - -In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an -attribute, it is said to be a :term:`descriptor`. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the method resolution order of ``type(a)``. If the -looked-up value is an object defining one of the descriptor methods, then Python -may override the default behavior and invoke the descriptor method instead. -Where this occurs in the precedence chain depends on which descriptor methods -were defined. - -Descriptors are a powerful, general purpose protocol. They are the mechanism -behind properties, methods, static methods, class methods, and -:func:`super()`. They are used throughout Python itself. Descriptors -simplify the underlying C code and offer a flexible set of new tools for -everyday Python programs. - - -Descriptor protocol -------------------- - -``descr.__get__(self, obj, type=None) -> value`` - -``descr.__set__(self, obj, value) -> None`` - -``descr.__delete__(self, obj) -> None`` - -That is all there is to it. Define any of these methods and an object is -considered a descriptor and can override default behavior upon being looked up -as an attribute. - -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are often used for methods but other uses are -possible). - -Data and non-data descriptors differ in how overrides are calculated with -respect to entries in an instance's dictionary. If an instance's dictionary -has an entry with the same name as a data descriptor, the data descriptor -takes precedence. If an instance's dictionary has an entry with the same -name as a non-data descriptor, the dictionary entry takes precedence. - -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising -placeholder is enough to make it a data descriptor. - - -Overview of descriptor invocation ---------------------------------- - -A descriptor can be called directly with ``desc.__get__(obj)`` or -``desc.__get__(None, cls)``. - -But it is more common for a descriptor to be invoked automatically from -attribute access. - -The expression ``obj.x`` looks up the attribute ``x`` in the chain of -namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. - -The details of invocation depend on whether ``obj`` is an object, class, or -instance of super. - - -Invocation from an instance ---------------------------- - -Instance lookup scans through a chain of namespaces giving data descriptors -the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is -provided. - -If a descriptor is found for ``a.x``, then it is invoked with: -``desc.__get__(a, type(a))``. - -The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is -a pure Python equivalent: - -.. testcode:: - - def object_getattribute(obj, name): - "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() - objtype = type(obj) - cls_var = getattr(objtype, name, null) - descr_get = getattr(type(cls_var), '__get__', null) - if descr_get is not null: - if (hasattr(type(cls_var), '__set__') - or hasattr(type(cls_var), '__delete__')): - return descr_get(cls_var, obj, objtype) # data descriptor - if hasattr(obj, '__dict__') and name in vars(obj): - return vars(obj)[name] # instance variable - if descr_get is not null: - return descr_get(cls_var, obj, objtype) # non-data descriptor - if cls_var is not null: - return cls_var # class variable - raise AttributeError(name) - - -.. testcode:: - :hide: - - # Test the fidelity of object_getattribute() by comparing it with the - # normal object.__getattribute__(). The former will be accessed by - # square brackets and the latter by the dot operator. - - class Object: - - def __getitem__(obj, name): - try: - return object_getattribute(obj, name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - - class DualOperator(Object): - - x = 10 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - @property - def p3(self): - return 3 * self.x - - def m5(self, y): - return 5 * y - - def m7(self, y): - return 7 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - class DualOperatorWithSlots: - - __getitem__ = Object.__getitem__ - - __slots__ = ['z'] - - x = 15 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - def m5(self, y): - return 5 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - -.. doctest:: - :hide: - - >>> a = DualOperator(11) - >>> vars(a).update(p3 = '_p3', m7 = '_m7') - >>> a.x == a['x'] == 10 - True - >>> a.z == a['z'] == 11 - True - >>> a.p2 == a['p2'] == 20 - True - >>> a.p3 == a['p3'] == 30 - True - >>> a.m5(100) == a.m5(100) == 500 - True - >>> a.m7 == a['m7'] == '_m7' - True - >>> a.g == a['g'] == ('getattr_hook', a, 'g') - True - - >>> b = DualOperatorWithSlots(22) - >>> b.x == b['x'] == 15 - True - >>> b.z == b['z'] == 22 - True - >>> b.p2 == b['p2'] == 30 - True - >>> b.m5(200) == b['m5'](200) == 1000 - True - >>> b.g == b['g'] == ('getattr_hook', b, 'g') - True - - -Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` -directly. Instead, both the dot operator and the :func:`getattr` function -perform attribute lookup by way of a helper function: - -.. testcode:: - - def getattr_hook(obj, name): - "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" - try: - return obj.__getattribute__(name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - -So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` -raises :exc:`AttributeError` (either directly or in one of the descriptor calls). - -Also, if a user calls :meth:`object.__getattribute__` directly, the -:meth:`__getattr__` hook is bypassed entirely. - - -Invocation from a class ------------------------ - -The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced -by a search through the class's :term:`method resolution order`. - -If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. - -The full C implementation can be found in :c:func:`type_getattro()` and -:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. - - -Invocation from super ---------------------- - -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. - -A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` -for the base class ``B`` immediately following ``A`` and then returns -``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned -unchanged. - -The full C implementation can be found in :c:func:`super_getattro()` in -:source:`Objects/typeobject.c`. A pure Python equivalent can be found in -`Guido's Tutorial -`_. - - -Summary of invocation logic ---------------------------- - -The mechanism for descriptors is embedded in the :meth:`__getattribute__()` -methods for :class:`object`, :class:`type`, and :func:`super`. - -The important points to remember are: - -* Descriptors are invoked by the :meth:`__getattribute__` method. - -* Classes inherit this machinery from :class:`object`, :class:`type`, or - :func:`super`. - -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls - because all the descriptor logic is in that method. - -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may - include the class. The second puts in ``None`` for the instance and always - includes the class. - -* Data descriptors always override instance dictionaries. - -* Non-data descriptors may be overridden by instance dictionaries. - - -Automatic name notification ---------------------------- - -Sometimes it is desirable for a descriptor to know what class variable name it -was assigned to. When a new class is created, the :class:`type` metaclass -scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two -arguments. The *owner* is the class where the descriptor is used, and the -*name* is the class variable the descriptor was assigned to. - -The implementation details are in :c:func:`type_new()` and -:c:func:`set_names()` in :source:`Objects/typeobject.c`. - -Since the update logic is in :meth:`type.__new__`, notifications only take -place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. - - -ORM example ------------ - -The following code is simplified skeleton showing how data descriptors could -be used to implement an `object relational mapping -`_. - -The essential idea is that the data is stored in an external database. The -Python instances only hold keys to the database's tables. Descriptors take -care of lookups or updates: - -.. testcode:: - - class Field: - - def __set_name__(self, owner, name): - self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' - self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' - - def __get__(self, obj, objtype=None): - return conn.execute(self.fetch, [obj.key]).fetchone()[0] - - def __set__(self, obj, value): - conn.execute(self.store, [value, obj.key]) - conn.commit() - -We can use the :class:`Field` class to define `models -`_ that describe the schema for -each table in a database: - -.. testcode:: - - class Movie: - table = 'Movies' # Table name - key = 'title' # Primary key - director = Field() - year = Field() - - def __init__(self, key): - self.key = key - - class Song: - table = 'Music' - key = 'title' - artist = Field() - year = Field() - genre = Field() - - def __init__(self, key): - self.key = key - -To use the models, first connect to the database:: - - >>> import sqlite3 - >>> conn = sqlite3.connect('entertainment.db') - -An interactive session shows how data is retrieved from the database and how -it can be updated: - -.. testsetup:: - - song_data = [ - ('Country Roads', 'John Denver', 1972), - ('Me and Bobby McGee', 'Janice Joplin', 1971), - ('Coal Miners Daughter', 'Loretta Lynn', 1970), - ] - - movie_data = [ - ('Star Wars', 'George Lucas', 1977), - ('Jaws', 'Steven Spielberg', 1975), - ('Aliens', 'James Cameron', 1986), - ] - - import sqlite3 - - conn = sqlite3.connect(':memory:') - conn.execute('CREATE TABLE Music (title text, artist text, year integer);') - conn.execute('CREATE INDEX MusicNdx ON Music (title);') - conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) - conn.execute('CREATE TABLE Movies (title text, director text, year integer);') - conn.execute('CREATE INDEX MovieNdx ON Music (title);') - conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) - conn.commit() - -.. doctest:: - - >>> Movie('Star Wars').director - 'George Lucas' - >>> jaws = Movie('Jaws') - >>> f'Released in {jaws.year} by {jaws.director}' - 'Released in 1975 by Steven Spielberg' - - >>> Song('Country Roads').artist - 'John Denver' - - >>> Movie('Star Wars').director = 'J.J. Abrams' - >>> Movie('Star Wars').director - 'J.J. Abrams' - - -Pure Python Equivalents -^^^^^^^^^^^^^^^^^^^^^^^ - -The descriptor protocol is simple and offers exciting possibilities. Several -use cases are so common that they have been prepackaged into built-in tools. -Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are -all based on the descriptor protocol. - - -Properties ----------- - -Calling :func:`property` is a succinct way of building a data descriptor that -triggers a function call upon access to an attribute. Its signature is:: - - property(fget=None, fset=None, fdel=None, doc=None) -> property - -The documentation shows a typical use to define a managed attribute ``x``: - -.. testcode:: - - class C: - def getx(self): return self.__x - def setx(self, value): self.__x = value - def delx(self): del self.__x - x = property(getx, setx, delx, "I'm the 'x' property.") - -To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent: - -.. testcode:: - - class Property: - "Emulate PyProperty_Type() in Objects/descrobject.c" - - def __init__(self, fget=None, fset=None, fdel=None, doc=None): - self.fget = fget - self.fset = fset - self.fdel = fdel - if doc is None and fget is not None: - doc = fget.__doc__ - self.__doc__ = doc - self._name = '' - - def __set_name__(self, owner, name): - self._name = name - - def __get__(self, obj, objtype=None): - if obj is None: - return self - if self.fget is None: - raise AttributeError(f'unreadable attribute {self._name}') - return self.fget(obj) - - def __set__(self, obj, value): - if self.fset is None: - raise AttributeError(f"can't set attribute {self._name}") - self.fset(obj, value) - - def __delete__(self, obj): - if self.fdel is None: - raise AttributeError(f"can't delete attribute {self._name}") - self.fdel(obj) - - def getter(self, fget): - prop = type(self)(fget, self.fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def setter(self, fset): - prop = type(self)(self.fget, fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def deleter(self, fdel): - prop = type(self)(self.fget, self.fset, fdel, self.__doc__) - prop._name = self._name - return prop - -.. testcode:: - :hide: - - # Verify the Property() emulation - - class CC: - def getx(self): - return self.__x - def setx(self, value): - self.__x = value - def delx(self): - del self.__x - x = Property(getx, setx, delx, "I'm the 'x' property.") - - # Now do it again but use the decorator style - - class CCC: - @Property - def x(self): - return self.__x - @x.setter - def x(self, value): - self.__x = value - @x.deleter - def x(self): - del self.__x - - -.. doctest:: - :hide: - - >>> cc = CC() - >>> hasattr(cc, 'x') - False - >>> cc.x = 33 - >>> cc.x - 33 - >>> del cc.x - >>> hasattr(cc, 'x') - False - - >>> ccc = CCC() - >>> hasattr(ccc, 'x') - False - >>> ccc.x = 333 - >>> ccc.x == 333 - True - >>> del ccc.x - >>> hasattr(ccc, 'x') - False - -The :func:`property` builtin helps whenever a user interface has granted -attribute access and then subsequent changes require the intervention of a -method. - -For instance, a spreadsheet class may grant access to a cell value through -``Cell('b10').value``. Subsequent improvements to the program require the cell -to be recalculated on every access; however, the programmer does not want to -affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor: - -.. testcode:: - - class Cell: - ... - - @property - def value(self): - "Recalculate the cell before returning value" - self.recalc() - return self._value - -Either the built-in :func:`property` or our :func:`Property` equivalent would -work in this example. - - -Functions and methods ---------------------- - -Python's object oriented features are built upon a function based environment. -Using non-data descriptors, the two are merged seamlessly. - -Functions stored in class dictionaries get turned into methods when invoked. -Methods only differ from regular functions in that the object instance is -prepended to the other arguments. By convention, the instance is called -*self* but could be called *this* or any other variable name. - -Methods can be created manually with :class:`types.MethodType` which is -roughly equivalent to: - -.. testcode:: - - class MethodType: - "Emulate Py_MethodType in Objects/classobject.c" - - def __init__(self, func, obj): - self.__func__ = func - self.__self__ = obj - - def __call__(self, *args, **kwargs): - func = self.__func__ - obj = self.__self__ - return func(obj, *args, **kwargs) - -To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This -means that functions are non-data descriptors that return bound methods -during dotted lookup from an instance. Here's how it works: - -.. testcode:: - - class Function: - ... - - def __get__(self, obj, objtype=None): - "Simulate func_descr_get() in Objects/funcobject.c" - if obj is None: - return self - return MethodType(self, obj) - -Running the following class in the interpreter shows how the function -descriptor works in practice: - -.. testcode:: - - class D: - def f(self, x): - return x - -The function has a :term:`qualified name` attribute to support introspection: - -.. doctest:: - - >>> D.f.__qualname__ - 'D.f' - -Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: - - >>> D.__dict__['f'] - - -Dotted access from a class calls :meth:`__get__` which just returns the -underlying function unchanged:: - - >>> D.f - - -The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: - - >>> d = D() - >>> d.f - > - -Internally, the bound method stores the underlying function and the bound -instance:: - - >>> d.f.__func__ - - - >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> - -If you have ever wondered where *self* comes from in regular methods or where -*cls* comes from in class methods, this is it! - - -Static methods --------------- - -Non-data descriptors provide a simple mechanism for variations on the usual -patterns of binding functions into methods. - -To recap, functions have a :meth:`__get__` method so that they can be converted -to a method when accessed as attributes. The non-data descriptor transforms an -``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` -becomes ``f(*args)``. - -This chart summarizes the binding and its two most useful variants: - - +-----------------+----------------------+------------------+ - | Transformation | Called from an | Called from a | - | | object | class | - +=================+======================+==================+ - | function | f(obj, \*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | staticmethod | f(\*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | classmethod | f(type(obj), \*args) | f(cls, \*args) | - +-----------------+----------------------+------------------+ - -Static methods return the underlying function without changes. Calling either -``c.f`` or ``C.f`` is the equivalent of a direct lookup into -``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a -result, the function becomes identically accessible from either an object or a -class. - -Good candidates for static methods are methods that do not reference the -``self`` variable. - -For instance, a statistics package may include a container class for -experimental data. The class provides normal methods for computing the average, -mean, median, and other descriptive statistics that depend on the data. However, -there may be useful functions which are conceptually related but do not depend -on the data. For instance, ``erf(x)`` is handy conversion routine that comes up -in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. - -Since static methods return the underlying function with no changes, the -example calls are unexciting: - -.. testcode:: - - class E: - @staticmethod - def f(x): - print(x) - -.. doctest:: - - >>> E.f(3) - 3 - >>> E().f(3) - 3 - -Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: - -.. doctest:: - - class StaticMethod: - "Emulate PyStaticMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, objtype=None): - return self.f - - -Class methods -------------- - -Unlike static methods, class methods prepend the class reference to the -argument list before calling the function. This format is the same -for whether the caller is an object or a class: - -.. testcode:: - - class F: - @classmethod - def f(cls, x): - return cls.__name__, x - -.. doctest:: - - >>> F.f(3) - ('F', 3) - >>> F().f(3) - ('F', 3) - -This behavior is useful whenever the method only needs to have a class -reference and does not rely on data stored in a specific instance. One use for -class methods is to create alternate class constructors. For example, the -classmethod :func:`dict.fromkeys` creates a new dictionary from a list of -keys. The pure Python equivalent is: - -.. testcode:: - - class Dict(dict): - @classmethod - def fromkeys(cls, iterable, value=None): - "Emulate dict_fromkeys() in Objects/dictobject.c" - d = cls() - for key in iterable: - d[key] = value - return d - -Now a new dictionary of unique keys can be constructed like this: - -.. doctest:: - - >>> d = Dict.fromkeys('abracadabra') - >>> type(d) is Dict - True - >>> d - {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} - -Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: - -.. testcode:: - - class ClassMethod: - "Emulate PyClassMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, cls=None): - if cls is None: - cls = type(obj) - if hasattr(obj, '__get__'): - return self.f.__get__(cls) - return MethodType(self.f, cls) - -.. testcode:: - :hide: - - # Verify the emulation works - class T: - @ClassMethod - def cm(cls, x, y): - return (cls, x, y) - -.. doctest:: - :hide: - - >>> T.cm(11, 22) - (, 11, 22) - - # Also call it from an instance - >>> t = T() - >>> t.cm(11, 22) - (, 11, 22) - -The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and -makes it possible for :func:`classmethod` to support chained decorators. -For example, a classmethod and property could be chained together: - -.. testcode:: - - class G: - @classmethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - -.. doctest:: - - >>> G.__doc__ - "A doc for 'G'" - - -Member objects and __slots__ ----------------------------- - -When a class defines ``__slots__``, it replaces instance dictionaries with a -fixed-length array of slot values. From a user point of view that has -several effects: - -1. Provides immediate detection of bugs due to misspelled attribute -assignments. Only attribute names specified in ``__slots__`` are allowed: - -.. testcode:: - - class Vehicle: - __slots__ = ('id_number', 'make', 'model') - -.. doctest:: - - >>> auto = Vehicle() - >>> auto.id_nubmer = 'VYE483814LQEX' - Traceback (most recent call last): - ... - AttributeError: 'Vehicle' object has no attribute 'id_nubmer' - -2. Helps create immutable objects where descriptors manage access to private -attributes stored in ``__slots__``: - -.. testcode:: - - class Immutable: - - __slots__ = ('_dept', '_name') # Replace the instance dictionary - - def __init__(self, dept, name): - self._dept = dept # Store to private attribute - self._name = name # Store to private attribute - - @property # Read-only descriptor - def dept(self): - return self._dept - - @property - def name(self): # Read-only descriptor - return self._name - -.. doctest:: - - >>> mark = Immutable('Botany', 'Mark Watney') - >>> mark.dept - 'Botany' - >>> mark.dept = 'Space Pirate' - Traceback (most recent call last): - ... - AttributeError: can't set attribute - >>> mark.location = 'Mars' - Traceback (most recent call last): - ... - AttributeError: 'Immutable' object has no attribute 'location' - -3. Saves memory. On a 64-bit Linux build, an instance with two attributes -takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight -design pattern `_ likely only -matters when a large number of instances are going to be created. - -4. Improves speed. Reading instance variables is 35% faster with -``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). - -5. Blocks tools like :func:`functools.cached_property` which require an -instance dictionary to function correctly: - -.. testcode:: - - from functools import cached_property - - class CP: - __slots__ = () # Eliminates the instance dict - - @cached_property # Requires an instance dict - def pi(self): - return 4 * sum((-1.0)**n / (2.0*n + 1.0) - for n in reversed(range(100_000))) - -.. doctest:: - - >>> CP().pi - Traceback (most recent call last): - ... - TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. - -It is not possible to create an exact drop-in pure Python version of -``__slots__`` because it requires direct access to C structures and control -over object memory allocation. However, we can build a mostly faithful -simulation where the actual C structure for slots is emulated by a private -``_slotvalues`` list. Reads and writes to that private structure are managed -by member descriptors: - -.. testcode:: - - null = object() - - class Member: - - def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' - # Also see descr_new() in Objects/descrobject.c - self.name = name - self.clsname = clsname - self.offset = offset - - def __get__(self, obj, objtype=None): - 'Emulate member_get() in Objects/descrobject.c' - # Also see PyMember_GetOne() in Python/structmember.c - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - return value - - def __set__(self, obj, value): - 'Emulate member_set() in Objects/descrobject.c' - obj._slotvalues[self.offset] = value - - def __delete__(self, obj): - 'Emulate member_delete() in Objects/descrobject.c' - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - obj._slotvalues[self.offset] = null - - def __repr__(self): - 'Emulate member_repr() in Objects/descrobject.c' - return f'' - -The :meth:`type.__new__` method takes care of adding member objects to class -variables: - -.. testcode:: - - class Type(type): - 'Simulate how the type metaclass adds member objects for slots' - - def __new__(mcls, clsname, bases, mapping): - 'Emuluate type_new() in Objects/typeobject.c' - # type_new() calls PyTypeReady() which calls add_methods() - slot_names = mapping.get('slot_names', []) - for offset, name in enumerate(slot_names): - mapping[name] = Member(name, clsname, offset) - return type.__new__(mcls, clsname, bases, mapping) - -The :meth:`object.__new__` method takes care of creating instances that have -slots instead of an instance dictionary. Here is a rough simulation in pure -Python: - -.. testcode:: - - class Object: - 'Simulate how object.__new__() allocates memory for __slots__' - - def __new__(cls, *args): - 'Emulate object_new() in Objects/typeobject.c' - inst = super().__new__(cls) - if hasattr(cls, 'slot_names'): - empty_slots = [null] * len(cls.slot_names) - object.__setattr__(inst, '_slotvalues', empty_slots) - return inst - - def __setattr__(self, name, value): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' - ) - super().__setattr__(name, value) - - def __delattr__(self, name): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' - ) - super().__delattr__(name) - -To use the simulation in a real class, just inherit from :class:`Object` and -set the :term:`metaclass` to :class:`Type`: - -.. testcode:: - - class H(Object, metaclass=Type): - 'Instance variables stored in slots' - - slot_names = ['x', 'y'] - - def __init__(self, x, y): - self.x = x - self.y = y - -At this point, the metaclass has loaded member objects for *x* and *y*:: - - >>> from pprint import pp - >>> pp(dict(vars(H))) - {'__module__': '__main__', - '__doc__': 'Instance variables stored in slots', - 'slot_names': ['x', 'y'], - '__init__': , - 'x': , - 'y': } - -.. doctest:: - :hide: - - # We test this separately because the preceding section is not - # doctestable due to the hex memory address for the __init__ function - >>> isinstance(vars(H)['x'], Member) - True - >>> isinstance(vars(H)['y'], Member) - True - -When instances are created, they have a ``slot_values`` list where the -attributes are stored: - -.. doctest:: - - >>> h = H(10, 20) - >>> vars(h) - {'_slotvalues': [10, 20]} - >>> h.x = 55 - >>> vars(h) - {'_slotvalues': [55, 20]} - -Misspelled or unassigned attributes will raise an exception: - -.. doctest:: - - >>> h.xz - Traceback (most recent call last): - ... - AttributeError: 'H' object has no attribute 'xz' - -.. doctest:: - :hide: - - # Examples for deleted attributes are not shown because this section - # is already a bit lengthy. We still test that code here. - >>> del h.x - >>> hasattr(h, 'x') - False - - # Also test the code for uninitialized slots - >>> class HU(Object, metaclass=Type): - ... slot_names = ['x', 'y'] - ... - >>> hu = HU() - >>> hasattr(hu, 'x') - False - >>> hasattr(hu, 'y') - False diff --git a/Doc/library/_thread.rst.bak b/Doc/library/_thread.rst.bak deleted file mode 100644 index bd653ab32bb9c43..000000000000000 --- a/Doc/library/_thread.rst.bak +++ /dev/null @@ -1,215 +0,0 @@ -:mod:`_thread` --- Low-level threading API -========================================== - -.. module:: _thread - :synopsis: Low-level threading API. - -.. index:: - single: light-weight processes - single: processes, light-weight - single: binary semaphores - single: semaphores, binary - --------------- - -This module provides low-level primitives for working with multiple threads -(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of -control sharing their global data space. For synchronization, simple locks -(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. -The :mod:`threading` module provides an easier to use and higher-level -threading API built on top of this module. - -.. index:: - single: pthreads - pair: threads; POSIX - -.. versionchanged:: 3.7 - This module used to be optional, it is now always available. - -This module defines the following constants and functions: - -.. exception:: error - - Raised on thread-specific errors. - - .. versionchanged:: 3.3 - This is now a synonym of the built-in :exc:`RuntimeError`. - - -.. data:: LockType - - This is the type of lock objects. - - -.. function:: start_new_thread(function, args[, kwargs]) - - Start a new thread and return its identifier. The thread executes the - function *function* with the argument list *args* (which must be a tuple). - The optional *kwargs* argument specifies a dictionary of keyword arguments. - - When the function returns, the thread silently exits. - - When the function terminates with an unhandled exception, - :func:`sys.unraisablehook` is called to handle the exception. The *object* - attribute of the hook argument is *function*. By default, a stack trace is - printed and then the thread exits (but other threads continue to run). - - When the function raises a :exc:`SystemExit` exception, it is silently - ignored. - - .. versionchanged:: 3.8 - :func:`sys.unraisablehook` is now used to handle unhandled exceptions. - - -.. function:: interrupt_main() - - Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main - thread. A thread can use this function to interrupt the main thread. - - If :data:`signal.SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. - - -.. function:: exit() - - Raise the :exc:`SystemExit` exception. When not caught, this will cause the - thread to exit silently. - -.. - function:: exit_prog(status) - - Exit all threads and report the value of the integer argument - *status* as the exit status of the entire program. - **Caveat:** code in pending :keyword:`finally` clauses, in this thread - or in other threads, is not executed. - - -.. function:: allocate_lock() - - Return a new lock object. Methods of locks are described below. The lock is - initially unlocked. - - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie to - be used e.g. to index a dictionary of thread-specific data. Thread identifiers - may be recycled when a thread exits and another thread is created. - - -.. function:: get_native_id() - - Return the native integral Thread ID of the current thread assigned by the kernel. - This is a non-negative integer. - Its value may be used to uniquely identify this particular thread system-wide - (until the thread terminates, after which the value may be recycled by the OS). - - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. - - .. versionadded:: 3.8 - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - - .. availability:: Windows, systems with POSIX threads. - - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire`. Specifying a timeout greater than this value will - raise an :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -Lock objects have the following methods: - - -.. method:: lock.acquire(waitflag=1, timeout=-1) - - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). - - If the integer *waitflag* argument is present, the action depends on its - value: if it is zero, the lock is only acquired if it can be acquired - immediately without waiting, while if it is nonzero, the lock is acquired - unconditionally as above. - - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *waitflag* is zero. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. - - -.. method:: lock.release() - - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. - - -.. method:: lock.locked() - - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. - -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: - - import _thread - - a_lock = _thread.allocate_lock() - - with a_lock: - print("a_lock is locked while this executes") - -**Caveats:** - - .. index:: module: signal - -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) - -* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is - equivalent to calling :func:`_thread.exit`. - -* It is not possible to interrupt the :meth:`acquire` method on a lock --- the - :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. - -* When the main thread exits, it is system defined whether the other threads - survive. On most systems, they are killed without executing - :keyword:`try` ... :keyword:`finally` clauses or executing object - destructors. - -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/ast.rst.bak b/Doc/library/ast.rst.bak deleted file mode 100644 index b5e1be826911575..000000000000000 --- a/Doc/library/ast.rst.bak +++ /dev/null @@ -1,1993 +0,0 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== - -.. module:: ast - :synopsis: Abstract Syntax Tree classes and manipulation. - -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - -.. testsetup:: - - import ast - -**Source code:** :source:`Lib/ast.py` - --------------- - -The :mod:`ast` module helps Python applications to process trees of the Python -abstract syntax grammar. The abstract syntax itself might change with each -Python release; this module helps to find out programmatically what the current -grammar looks like. - -An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as -a flag to the :func:`compile` built-in function, or using the :func:`parse` -helper provided in this module. The result will be a tree of objects whose -classes all inherit from :class:`ast.AST`. An abstract syntax tree can be -compiled into a Python code object using the built-in :func:`compile` function. - - -.. _abstract-grammar: - -Abstract Grammar ----------------- - -The abstract grammar is currently defined as follows: - -.. literalinclude:: ../../Parser/Python.asdl - :language: asdl - - -Node classes ------------- - -.. class:: AST - - This is the base of all AST node classes. The actual node classes are - derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`below `. They are defined in the :mod:`_ast` C - module and re-exported in :mod:`ast`. - - There is one class defined for each left-hand side symbol in the abstract - grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, - there is one class defined for each constructor on the right-hand side; these - classes inherit from the classes for the left-hand side trees. For example, - :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules - with alternatives (aka "sums"), the left-hand side class is abstract: only - instances of specific constructor nodes are ever created. - - .. index:: single: ? (question mark); in AST grammar - .. index:: single: * (asterisk); in AST grammar - - .. attribute:: _fields - - Each concrete class has an attribute :attr:`_fields` which gives the names - of all child nodes. - - Each instance of a concrete class has one attribute for each child node, - of the type as defined in the grammar. For example, :class:`ast.BinOp` - instances have an attribute :attr:`left` of type :class:`ast.expr`. - - If these attributes are marked as optional in the grammar (using a - question mark), the value might be ``None``. If the attributes can have - zero-or-more values (marked with an asterisk), the values are represented - as Python lists. All possible attributes must be present and have valid - values when compiling an AST with :func:`compile`. - - .. attribute:: lineno - col_offset - end_lineno - end_col_offset - - Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and - :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` - are the first and last line numbers of source text span (1-indexed so the - first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` - are the corresponding UTF-8 byte offsets of the first and last tokens that - generated the node. The UTF-8 offset is recorded because the parser uses - UTF-8 internally. - - Note that the end positions are not required by the compiler and are - therefore optional. The end offset is *after* the last symbol, for example - one can get the source segment of a one-line expression node using - ``source_line[node.col_offset : node.end_col_offset]``. - - The constructor of a class :class:`ast.T` parses its arguments as follows: - - * If there are positional arguments, there must be as many as there are items - in :attr:`T._fields`; they will be assigned as attributes of these names. - * If there are keyword arguments, they will set the attributes of the same - names to the given values. - - For example, to create and populate an :class:`ast.UnaryOp` node, you could - use :: - - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), - lineno=0, col_offset=0) - -.. versionchanged:: 3.8 - - Class :class:`ast.Constant` is now used for all constants. - -.. versionchanged:: 3.9 - - Simple indices are represented by their value, extended slices are - represented as tuples. - -.. deprecated:: 3.8 - - Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, - :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. - -.. deprecated:: 3.9 - - Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still - available, but they will be removed in future Python releases. - In the meantime, instantiating them will return an instance of - a different class. - -.. note:: - The descriptions of the specific node classes displayed here - were initially adapted from the fantastic `Green Tree - Snakes `__ project and - all its contributors. - -Literals -^^^^^^^^ - -.. class:: Constant(value) - - A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. - - .. doctest:: - - >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) - Expression( - body=Constant(value=123)) - - -.. class:: FormattedValue(value, conversion, format_spec) - - Node representing a single formatting field in an f-string. If the string - contains a single formatting field and nothing else the node can be - isolated otherwise it appears in :class:`JoinedStr`. - - * ``value`` is any expression node (such as a literal, a variable, or a - function call). - * ``conversion`` is an integer: - - * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting - - * ``format_spec`` is a :class:`JoinedStr` node representing the formatting - of the value, or ``None`` if no format was specified. Both - ``conversion`` and ``format_spec`` can be set at the same time. - - -.. class:: JoinedStr(values) - - An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` - nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) - Expression( - body=JoinedStr( - values=[ - Constant(value='sin('), - FormattedValue( - value=Name(id='a', ctx=Load()), - conversion=-1), - Constant(value=') is '), - FormattedValue( - value=Call( - func=Name(id='sin', ctx=Load()), - args=[ - Name(id='a', ctx=Load())], - keywords=[]), - conversion=-1, - format_spec=JoinedStr( - values=[ - Constant(value='.3')]))])) - - -.. class:: List(elts, ctx) - Tuple(elts, ctx) - - A list or tuple. ``elts`` holds a list of nodes representing the elements. - ``ctx`` is :class:`Store` if the container is an assignment target (i.e. - ``(x,y)=something``), and :class:`Load` otherwise. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) - Expression( - body=List( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) - Expression( - body=Tuple( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - - -.. class:: Set(elts) - - A set. ``elts`` holds a list of nodes representing the set's elements. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) - Expression( - body=Set( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)])) - - -.. class:: Dict(keys, values) - - A dictionary. ``keys`` and ``values`` hold lists of nodes representing the - keys and the values respectively, in matching order (what would be returned - when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). - - When doing dictionary unpacking using dictionary literals the expression to be - expanded goes in the ``values`` list, with a ``None`` at the corresponding - position in ``keys``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) - Expression( - body=Dict( - keys=[ - Constant(value='a'), - None], - values=[ - Constant(value=1), - Name(id='d', ctx=Load())])) - - -Variables -^^^^^^^^^ - -.. class:: Name(id, ctx) - - A variable name. ``id`` holds the name as a string, and ``ctx`` is one of - the following types. - - -.. class:: Load() - Store() - Del() - - Variable references can be used to load the value of a variable, to assign - a new value to it, or to delete it. Variable references are given a context - to distinguish these cases. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a'), indent=4)) - Module( - body=[ - Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a = 1'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('del a'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) - - -.. class:: Starred(value, ctx) - - A ``*var`` variable reference. ``value`` holds the variable, typically a - :class:`Name` node. This type must be used when building a :class:`Call` - node with ``*args``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Starred( - value=Name(id='b', ctx=Store()), - ctx=Store())], - ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) - - -Expressions -^^^^^^^^^^^ - -.. class:: Expr(value) - - When an expression, such as a function call, appears as a statement by itself - with its return value not used or stored, it is wrapped in this container. - ``value`` holds one of the other nodes in this section, a :class:`Constant`, a - :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('-a'), indent=4)) - Module( - body=[ - Expr( - value=UnaryOp( - op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) - - -.. class:: UnaryOp(op, operand) - - A unary operation. ``op`` is the operator, and ``operand`` any expression - node. - - -.. class:: UAdd - USub - Not - Invert - - Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` - is the ``~`` operator. - - .. doctest:: - - >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) - Expression( - body=UnaryOp( - op=Not(), - operand=Name(id='x', ctx=Load()))) - - -.. class:: BinOp(left, op, right) - - A binary operation (like addition or division). ``op`` is the operator, and - ``left`` and ``right`` are any expression nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) - Expression( - body=BinOp( - left=Name(id='x', ctx=Load()), - op=Add(), - right=Name(id='y', ctx=Load()))) - - -.. class:: Add - Sub - Mult - Div - FloorDiv - Mod - Pow - LShift - RShift - BitOr - BitXor - BitAnd - MatMult - - Binary operator tokens. - - -.. class:: BoolOp(op, values) - - A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. - ``values`` are the values involved. Consecutive operations with the same - operator, such as ``a or b or c``, are collapsed into one node with several - values. - - This doesn't include ``not``, which is a :class:`UnaryOp`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) - Expression( - body=BoolOp( - op=Or(), - values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) - - -.. class:: And - Or - - Boolean operator tokens. - - -.. class:: Compare(left, ops, comparators) - - A comparison of two or more values. ``left`` is the first value in the - comparison, ``ops`` the list of operators, and ``comparators`` the list - of values after the first element in the comparison. - - .. doctest:: - - >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) - Expression( - body=Compare( - left=Constant(value=1), - ops=[ - LtE(), - Lt()], - comparators=[ - Name(id='a', ctx=Load()), - Constant(value=10)])) - - -.. class:: Eq - NotEq - Lt - LtE - Gt - GtE - Is - IsNot - In - NotIn - - Comparison operator tokens. - - -.. class:: Call(func, args, keywords, starargs, kwargs) - - A function call. ``func`` is the function, which will often be a - :class:`Name` or :class:`Attribute` object. Of the arguments: - - * ``args`` holds a list of the arguments passed by position. - * ``keywords`` holds a list of :class:`keyword` objects representing - arguments passed by keyword. - - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. ``starargs`` and ``kwargs`` are optional. - - .. doctest:: - - >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) - Expression( - body=Call( - func=Name(id='func', ctx=Load()), - args=[ - Name(id='a', ctx=Load()), - Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], - keywords=[ - keyword( - arg='b', - value=Name(id='c', ctx=Load())), - keyword( - value=Name(id='e', ctx=Load()))])) - - -.. class:: keyword(arg, value) - - A keyword argument to a function call or class definition. ``arg`` is a raw - string of the parameter name, ``value`` is a node to pass in. - - -.. class:: IfExp(test, body, orelse) - - An expression such as ``a if b else c``. Each field holds a single node, so - in the following example, all three are :class:`Name` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) - Expression( - body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) - - -.. class:: Attribute(value, attr, ctx) - - Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a - :class:`Name`. ``attr`` is a bare string giving the name of the attribute, - and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how - the attribute is acted on. - - .. doctest:: - - >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) - Expression( - body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) - - -.. class:: NamedExpr(target, value) - - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) - Expression( - body=NamedExpr( - target=Name(id='x', ctx=Store()), - value=Constant(value=4))) - - -Subscripting -~~~~~~~~~~~~ - -.. class:: Subscript(value, slice, ctx) - - A subscript, such as ``l[1]``. ``value`` is the subscripted object - (usually sequence or mapping). ``slice`` is an index, slice or key. - It can be a :class:`Tuple` and contain a :class:`Slice`. - ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` - according to the action performed with the subscript. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Tuple( - elts=[ - Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) - - -.. class:: Slice(lower, upper, step) - - Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). - Can occur only inside the *slice* field of :class:`Subscript`, either - directly or as an element of :class:`Tuple`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) - - -Comprehensions -~~~~~~~~~~~~~~ - -.. class:: ListComp(elt, generators) - SetComp(elt, generators) - GeneratorExp(elt, generators) - DictComp(key, value, generators) - - List and set comprehensions, generator expressions, and dictionary - comprehensions. ``elt`` (or ``key`` and ``value``) is a single node - representing the part that will be evaluated for each item. - - ``generators`` is a list of :class:`comprehension` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) - Expression( - body=ListComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) - Expression( - body=DictComp( - key=Name(id='x', ctx=Load()), - value=BinOp( - left=Name(id='x', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) - Expression( - body=SetComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - - -.. class:: comprehension(target, iter, ifs, is_async) - - One ``for`` clause in a comprehension. ``target`` is the reference to use for - each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` - is the object to iterate over. ``ifs`` is a list of test expressions: each - ``for`` clause can have multiple ``ifs``. - - ``is_async`` indicates a comprehension is asynchronous (using an - ``async for`` instead of ``for``). The value is an integer (0 or 1). - - .. doctest:: - - >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), - ... indent=4)) # Multiple comprehensions in one. - Expression( - body=ListComp( - elt=Call( - func=Name(id='ord', ctx=Load()), - args=[ - Name(id='c', ctx=Load())], - keywords=[]), - generators=[ - comprehension( - target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), - ifs=[], - is_async=0), - comprehension( - target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), - ifs=[], - is_async=0)])) - - >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), - ... indent=4)) # generator comprehension - Expression( - body=GeneratorExp( - elt=BinOp( - left=Name(id='n', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), - ifs=[ - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Lt()], - comparators=[ - Constant(value=10)])], - is_async=0)])) - - >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), - ... indent=4)) # Async comprehension - Expression( - body=ListComp( - elt=Name(id='i', ctx=Load()), - generators=[ - comprehension( - target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), - ifs=[], - is_async=1)])) - -Statements -^^^^^^^^^^ - -.. class:: Assign(targets, value, type_comment) - - An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. - - Multiple nodes in ``targets`` represents assigning the same value to each. - Unpacking is represented by putting a :class:`Tuple` or :class:`List` - within ``targets``. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) - - -.. class:: AnnAssign(target, annotation, value, simple) - - An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. - ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. - - .. doctest:: - - >>> print(ast.dump(ast.parse('c: int'), indent=4)) - Module( - body=[ - AnnAssign( - target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis - Module( - body=[ - AnnAssign( - target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - value=Constant(value=1), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation - Module( - body=[ - AnnAssign( - target=Attribute( - value=Name(id='a', ctx=Load()), - attr='b', - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation - Module( - body=[ - AnnAssign( - target=Subscript( - value=Name(id='a', ctx=Load()), - slice=Constant(value=1), - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - -.. class:: AugAssign(target, op, value) - - Augmented assignment, such as ``a += 1``. In the following example, - ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` - context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with - value for 1. - - The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, - unlike the targets of :class:`Assign`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x += 2'), indent=4)) - Module( - body=[ - AugAssign( - target=Name(id='x', ctx=Store()), - op=Add(), - value=Constant(value=2))], - type_ignores=[]) - - -.. class:: Raise(exc, cause) - - A ``raise`` statement. ``exc`` is the exception object to be raised, normally a - :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. - ``cause`` is the optional part for ``y`` in ``raise x from y``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) - Module( - body=[ - Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Assert(test, msg) - - An assertion. ``test`` holds the condition, such as a :class:`Compare` node. - ``msg`` holds the failure message. - - .. doctest:: - - >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) - Module( - body=[ - Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Delete(targets) - - Represents a ``del`` statement. ``targets`` is a list of nodes, such as - :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='x', ctx=Del()), - Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) - - -.. class:: Pass() - - A ``pass`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('pass'), indent=4)) - Module( - body=[ - Pass()], - type_ignores=[]) - - -Other statements which are only applicable inside functions or loops are -described in other sections. - -Imports -~~~~~~~ - -.. class:: Import(names) - - An import statement. ``names`` is a list of :class:`alias` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) - Module( - body=[ - Import( - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')])], - type_ignores=[]) - - -.. class:: ImportFrom(module, names, level) - - Represents ``from x import y``. ``module`` is a raw string of the 'from' name, - without any leading dots, or ``None`` for statements such as ``from . import foo``. - ``level`` is an integer holding the level of the relative import (0 means - absolute import). - - .. doctest:: - - >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) - Module( - body=[ - ImportFrom( - module='y', - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')], - level=0)], - type_ignores=[]) - - -.. class:: alias(name, asname) - - Both parameters are raw strings of the names. ``asname`` can be ``None`` if - the regular name is to be used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) - Module( - body=[ - ImportFrom( - module='foo.bar', - names=[ - alias(name='a', asname='b'), - alias(name='c')], - level=2)], - type_ignores=[]) - -Control flow -^^^^^^^^^^^^ - -.. note:: - Optional clauses such as ``else`` are stored as an empty list if they're - not present. - -.. class:: If(test, body, orelse) - - An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` - node. ``body`` and ``orelse`` each hold a list of nodes. - - ``elif`` clauses don't have a special representation in the AST, but rather - appear as extra :class:`If` nodes within the ``orelse`` section of the - previous one. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... if x: - ... ... - ... elif y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - If( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - If( - test=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: For(target, iter, body, orelse, type_comment) - - A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a - single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds - the item to be looped over, again as a single node. ``body`` and ``orelse`` - contain lists of nodes to execute. Those in ``orelse`` are executed if the - loop finishes normally, rather than via a ``break`` statement. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... for x in y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: While(test, body, orelse) - - A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` - node. - - .. doctest:: - - >> print(ast.dump(ast.parse(""" - ... while x: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - While( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: Break - Continue - - The ``break`` and ``continue`` statements. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... for a in b: - ... if a > 5: - ... break - ... else: - ... continue - ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), - body=[ - If( - test=Compare( - left=Name(id='a', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - body=[ - Break()], - orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) - - -.. class:: Try(body, handlers, orelse, finalbody) - - ``try`` blocks. All attributes are list of nodes to execute, except for - ``handlers``, which is a list of :class:`ExceptHandler` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... try: - ... ... - ... except Exception: - ... ... - ... except OtherException as e: - ... ... - ... else: - ... ... - ... finally: - ... ... - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=Constant(value=Ellipsis))], - handlers=[ - ExceptHandler( - type=Name(id='Exception', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - ExceptHandler( - type=Name(id='OtherException', ctx=Load()), - name='e', - body=[ - Expr( - value=Constant(value=Ellipsis))])], - orelse=[ - Expr( - value=Constant(value=Ellipsis))], - finalbody=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: ExceptHandler(type, name, body) - - A single ``except`` clause. ``type`` is the exception type it will match, - typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). - ``name`` is a raw string for the name to hold the exception, or ``None`` if - the clause doesn't have ``as foo``. ``body`` is a list of nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... try: - ... a + 1 - ... except TypeError: - ... pass - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=BinOp( - left=Name(id='a', ctx=Load()), - op=Add(), - right=Constant(value=1)))], - handlers=[ - ExceptHandler( - type=Name(id='TypeError', ctx=Load()), - body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) - - -.. class:: With(items, body, type_comment) - - A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing - the context managers, and ``body`` is the indented block inside the context. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: withitem(context_expr, optional_vars) - - A single context manager in a ``with`` block. ``context_expr`` is the context - manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, - :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that - isn't used. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... with a as b, c as d: - ... something(b, d) - ... """), indent=4)) - Module( - body=[ - With( - items=[ - withitem( - context_expr=Name(id='a', ctx=Load()), - optional_vars=Name(id='b', ctx=Store())), - withitem( - context_expr=Name(id='c', ctx=Load()), - optional_vars=Name(id='d', ctx=Store()))], - body=[ - Expr( - value=Call( - func=Name(id='something', ctx=Load()), - args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) - - -.. class:: Match(subject, cases) - - A ``match`` statement. ``subject`` holds the subject of the match (the object - that is being matched against the cases) and ``cases`` contains an iterable of - :class:`match_case` nodes with the different cases. - - -.. class:: match_case(context_expr, optional_vars) - - A single case pattern in a ``match`` statement. ``pattern`` contains the - match pattern that will be used to match the subject against. Notice that - the meaning of the :class:`AST` nodes in this attribute have a different - meaning than in other places, as they represent patterns to match against. - The ``guard`` attribute contains an expression that will be evaluated if - the pattern matches the subject. If the pattern matches and the ``guard`` condition - is truthy, the body of the case shall be executed. ``body`` contains a list - of nodes to execute if the guard is truthy. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] if x>0: - ... ... - ... case tuple(): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - guard=Compare( - left=Name(id='x', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=0)]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=Call( - func=Name(id='tuple', ctx=Load()), - args=[], - keywords=[]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchAs(pattern, name) - - A match "as-pattern". The as-pattern matches whatever pattern is on its - left-hand side, but also binds the value to a name. ``pattern`` contains - the match pattern that will be used to match the subject agsinst. The ``name`` - attribute contains the name that will be binded if the pattern is successful. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] as y: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchAs( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - name='y'), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: MatchOr(patterns) - - A match "or-pattern". An or-pattern matches each of its subpatterns in turn - to the subject, until one succeeds. The or-pattern is then deemed to - succeed. If none of the subpatterns succeed the or-pattern fails. The - ``patterns`` attribute contains a list of match patterns nodes that will be - matched against the subject. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] | (y): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchOr( - patterns=[ - List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - Name(id='y', ctx=Store())]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -Function and class definitions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) - - A function definition. - - * ``name`` is a raw string of the function name. - * ``args`` is a :class:`arguments` node. - * ``body`` is the list of nodes inside the function. - * ``decorator_list`` is the list of decorators to be applied, stored outermost - first (i.e. the first in the list will be applied last). - * ``returns`` is the return annotation. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: Lambda(args, body) - - ``lambda`` is a minimal function definition that can be used inside an - expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) - Module( - body=[ - Expr( - value=Lambda( - args=arguments( - posonlyargs=[], - args=[ - arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) - - -.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) - - The arguments for a function. - - * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. - * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the - ``*args, **kwargs`` parameters. - * ``kw_defaults`` is a list of default values for keyword-only arguments. If - one is ``None``, the corresponding argument is required. - * ``defaults`` is a list of default values for arguments that can be passed - positionally. If there are fewer defaults, they correspond to the last n - arguments. - - -.. class:: arg(arg, annotation, type_comment) - - A single argument in a list. ``arg`` is a raw string of the argument - name, ``annotation`` is its annotation, such as a :class:`Str` or - :class:`Name` node. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': - ... pass - ... """), indent=4)) - Module( - body=[ - FunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[ - arg( - arg='a', - annotation=Constant(value='annotation')), - arg(arg='b'), - arg(arg='c')], - vararg=arg(arg='d'), - kwonlyargs=[ - arg(arg='e'), - arg(arg='f')], - kw_defaults=[ - None, - Constant(value=3)], - kwarg=arg(arg='g'), - defaults=[ - Constant(value=1), - Constant(value=2)]), - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'))], - type_ignores=[]) - - -.. class:: Return(value) - - A ``return`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('return 4'), indent=4)) - Module( - body=[ - Return( - value=Constant(value=4))], - type_ignores=[]) - - -.. class:: Yield(value) - YieldFrom(value) - - A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('yield x'), indent=4)) - Module( - body=[ - Expr( - value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('yield from x'), indent=4)) - Module( - body=[ - Expr( - value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - -.. class:: Global(names) - Nonlocal(names) - - ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. - - .. doctest:: - - >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) - Module( - body=[ - Global( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) - Module( - body=[ - Nonlocal( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - -.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) - - A class definition. - - * ``name`` is a raw string for the class name - * ``bases`` is a list of nodes for explicitly specified base classes. - * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. - Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. - * ``starargs`` and ``kwargs`` are each a single node, as in a function call. - starargs will be expanded to join the list of base classes, and kwargs will - be passed to the metaclass. - * ``body`` is a list of nodes representing the code within the class - definition. - * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... class Foo(base1, base2, metaclass=meta): - ... pass - ... """), indent=4)) - Module( - body=[ - ClassDef( - name='Foo', - bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], - keywords=[ - keyword( - arg='metaclass', - value=Name(id='meta', ctx=Load()))], - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])], - type_ignores=[]) - -Async and await -^^^^^^^^^^^^^^^ - -.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) - - An ``async def`` function definition. Has the same fields as - :class:`FunctionDef`. - - -.. class:: Await(value) - - An ``await`` expression. ``value`` is what it waits for. - Only valid in the body of an :class:`AsyncFunctionDef`. - -.. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[])], - type_ignores=[]) - - -.. class:: AsyncFor(target, iter, body, orelse, type_comment) - AsyncWith(items, body, type_comment) - - ``async for`` loops and ``async with`` context managers. They have the same - fields as :class:`For` and :class:`With`, respectively. Only valid in the - body of an :class:`AsyncFunctionDef`. - -.. note:: - When a string is parsed by :func:`ast.parse`, operator nodes (subclasses - of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, - :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree - will be singletons. Changes to one will be reflected in all other - occurrences of the same value (e.g. :class:`ast.Add`). - - -:mod:`ast` Helpers ------------------- - -Apart from the node classes, the :mod:`ast` module defines these utility functions -and classes for traversing abstract syntax trees: - -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) - - Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. - - If ``type_comments=True`` is given, the parser is modified to check - and return type comments as specified by :pep:`484` and :pep:`526`. - This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the - flags passed to :func:`compile()`. This will report syntax errors - for misplaced type comments. Without this flag, type comments will - be ignored, and the ``type_comment`` field on selected AST nodes - will always be ``None``. In addition, the locations of ``# type: - ignore`` comments will be returned as the ``type_ignores`` - attribute of :class:`Module` (otherwise it is always an empty list). - - In addition, if ``mode`` is ``'func_type'``, the input syntax is - modified to correspond to :pep:`484` "signature type comments", - e.g. ``(str, int) -> List[str]``. - - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 4)``; the highest is ``sys.version_info[0:2]``. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - .. versionchanged:: 3.8 - Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. - - -.. function:: unparse(ast_obj) - - Unparse an :class:`ast.AST` object and generate a string with code - that would produce an equivalent :class:`ast.AST` object if parsed - back with :func:`ast.parse`. - - .. warning:: - The produced code string will not necessarily be equal to the original - code that generated the :class:`ast.AST` object (without any compiler - optimizations, such as constant tuples/frozensets). - - .. warning:: - Trying to unparse a highly complex expression would result with - :exc:`RecursionError`. - - .. versionadded:: 3.9 - - -.. function:: literal_eval(node_or_string) - - Safely evaluate an expression node or a string containing a Python literal or - container display. The string or node provided may only consist of the - following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, ``None`` and ``Ellipsis``. - - This can be used for safely evaluating strings containing Python values from - untrusted sources without the need to parse the values oneself. It is not - capable of evaluating arbitrarily complex expressions, for example involving - operators or indexing. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, - :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed - input. - - .. versionchanged:: 3.2 - Now allows bytes and set literals. - - .. versionchanged:: 3.9 - Now supports creating empty sets with ``'set()'``. - - .. versionchanged:: 3.10 - For string inputs, leading spaces and tabs are now stripped. - - -.. function:: get_docstring(node, clean=True) - - Return the docstring of the given *node* (which must be a - :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, - or :class:`Module` node), or ``None`` if it has no docstring. - If *clean* is true, clean up the docstring's indentation with - :func:`inspect.cleandoc`. - - .. versionchanged:: 3.5 - :class:`AsyncFunctionDef` is now supported. - - -.. function:: get_source_segment(source, node, *, padded=False) - - Get source code segment of the *source* that generated *node*. - If some location information (:attr:`lineno`, :attr:`end_lineno`, - :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. - - If *padded* is ``True``, the first line of a multi-line statement will - be padded with spaces to match its original position. - - .. versionadded:: 3.8 - - -.. function:: fix_missing_locations(node) - - When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports - them. This is rather tedious to fill in for generated nodes, so this helper - adds these attributes recursively where not already set, by setting them to - the values of the parent node. It works recursively starting at *node*. - - -.. function:: increment_lineno(node, n=1) - - Increment the line number and end line number of each node in the tree - starting at *node* by *n*. This is useful to "move code" to a different - location in a file. - - -.. function:: copy_location(new_node, old_node) - - Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, - and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, - and return *new_node*. - - -.. function:: iter_fields(node) - - Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` - that is present on *node*. - - -.. function:: iter_child_nodes(node) - - Yield all direct child nodes of *node*, that is, all fields that are nodes - and all items of fields that are lists of nodes. - - -.. function:: walk(node) - - Recursively yield all descendant nodes in the tree starting at *node* - (including *node* itself), in no specified order. This is useful if you only - want to modify nodes in place and don't care about the context. - - -.. class:: NodeVisitor() - - A node visitor base class that walks the abstract syntax tree and calls a - visitor function for every node found. This function may return a value - which is forwarded by the :meth:`visit` method. - - This class is meant to be subclassed, with the subclass adding visitor - methods. - - .. method:: visit(node) - - Visit a node. The default implementation calls the method called - :samp:`self.visit_{classname}` where *classname* is the name of the node - class, or :meth:`generic_visit` if that method doesn't exist. - - .. method:: generic_visit(node) - - This visitor calls :meth:`visit` on all children of the node. - - Note that child nodes of nodes that have a custom visitor method won't be - visited unless the visitor calls :meth:`generic_visit` or visits them - itself. - - Don't use the :class:`NodeVisitor` if you want to apply changes to nodes - during traversal. For this a special visitor exists - (:class:`NodeTransformer`) that allows modifications. - - .. deprecated:: 3.8 - - Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, - :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. - - -.. class:: NodeTransformer() - - A :class:`NodeVisitor` subclass that walks the abstract syntax tree and - allows modification of nodes. - - The :class:`NodeTransformer` will walk the AST and use the return value of - the visitor methods to replace or remove the old node. If the return value - of the visitor method is ``None``, the node will be removed from its - location, otherwise it is replaced with the return value. The return value - may be the original node in which case no replacement takes place. - - Here is an example transformer that rewrites all occurrences of name lookups - (``foo``) to ``data['foo']``:: - - class RewriteName(NodeTransformer): - - def visit_Name(self, node): - return Subscript( - value=Name(id='data', ctx=Load()), - slice=Constant(value=node.id), - ctx=node.ctx - ) - - Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` - method for the node first. - - For nodes that were part of a collection of statements (that applies to all - statement nodes), the visitor may also return a list of nodes rather than - just a single node. - - If :class:`NodeTransformer` introduces new nodes (that weren't part of - original tree) without giving them location information (such as - :attr:`lineno`), :func:`fix_missing_locations` should be called with - the new sub-tree to recalculate the location information:: - - tree = ast.parse('foo', mode='eval') - new_tree = fix_missing_locations(RewriteName().visit(tree)) - - Usually you use the transformer like this:: - - node = YourTransformer().visit(node) - - -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) - - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. If *annotate_fields* is true (by default), - the returned string will show the names and the values for fields. - If *annotate_fields* is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to true. - - If *indent* is a non-negative integer or string, then the tree will be - pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the single line representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.9 - Added the *indent* option. - - -.. _ast-compiler-flags: - -Compiler Flags --------------- - -The following flags may be passed to :func:`compile` in order to change -effects on the compilation of a program: - -.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT - - Enables support for top-level ``await``, ``async for``, ``async with`` - and async comprehensions. - - .. versionadded:: 3.8 - -.. data:: PyCF_ONLY_AST - - Generates and returns an abstract syntax tree instead of returning a - compiled code object. - -.. data:: PyCF_TYPE_COMMENTS - - Enables support for :pep:`484` and :pep:`526` style type comments - (``# type: ``, ``# type: ignore ``). - - .. versionadded:: 3.8 - - -.. _ast-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 3.9 - -The :mod:`ast` module can be executed as a script from the command line. -It is as simple as: - -.. code-block:: sh - - python -m ast [-m ] [-a] [infile] - -The following options are accepted: - -.. program:: ast - -.. cmdoption:: -h, --help - - Show the help message and exit. - -.. cmdoption:: -m - --mode - - Specify what kind of code must be compiled, like the *mode* argument - in :func:`parse`. - -.. cmdoption:: --no-type-comments - - Don't parse type comments. - -.. cmdoption:: -a, --include-attributes - - Include attributes such as line numbers and column offsets. - -.. cmdoption:: -i - --indent - - Indentation of nodes in AST (number of spaces). - -If :file:`infile` is specified its contents are parsed to AST and dumped -to stdout. Otherwise, the content is read from stdin. - - -.. seealso:: - - `Green Tree Snakes `_, an external - documentation resource, has good details on working with Python ASTs. - - `ASTTokens `_ - annotates Python ASTs with the positions of tokens and text in the source - code that generated them. This is helpful for tools that make source code - transformations. - - `leoAst.py `_ unifies the - token-based and parse-tree-based views of python programs by inserting - two-way links between tokens and ast nodes. - - `LibCST `_ parses code as a Concrete Syntax - Tree that looks like an ast tree and keeps all formatting details. It's - useful for building automated refactoring (codemod) applications and - linters. - - `Parso `_ is a Python parser that supports - error recovery and round-trip parsing for different Python versions (in - multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. diff --git a/Doc/library/asyncio-api-index.rst.bak b/Doc/library/asyncio-api-index.rst.bak deleted file mode 100644 index 047e5bbc58ccade..000000000000000 --- a/Doc/library/asyncio-api-index.rst.bak +++ /dev/null @@ -1,221 +0,0 @@ -.. currentmodule:: asyncio - - -==================== -High-level API Index -==================== - -This page lists all high-level async/await enabled asyncio APIs. - - -Tasks -===== - -Utilities to run asyncio programs, create Tasks, and -await on multiple things with timeouts. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :func:`run` - - Create event loop, run a coroutine, close the loop. - - * - :func:`create_task` - - Start an asyncio Task. - - * - ``await`` :func:`sleep` - - Sleep for a number of seconds. - - * - ``await`` :func:`gather` - - Schedule and wait for things concurrently. - - * - ``await`` :func:`wait_for` - - Run with a timeout. - - * - ``await`` :func:`shield` - - Shield from cancellation. - - * - ``await`` :func:`wait` - - Monitor for completion. - - * - :func:`current_task` - - Return the current Task. - - * - :func:`all_tasks` - - Return all tasks for an event loop. - - * - :class:`Task` - - Task object. - - * - :func:`to_thread` - - Asychronously run a function in a separate OS thread. - - * - :func:`run_coroutine_threadsafe` - - Schedule a coroutine from another OS thread. - - * - ``for in`` :func:`as_completed` - - Monitor for completion with a ``for`` loop. - - -.. rubric:: Examples - -* :ref:`Using asyncio.gather() to run things in parallel - `. - -* :ref:`Using asyncio.wait_for() to enforce a timeout - `. - -* :ref:`Cancellation `. - -* :ref:`Using asyncio.sleep() `. - -* See also the main :ref:`Tasks documentation page `. - - -Queues -====== - -Queues should be used to distribute work amongst multiple asyncio Tasks, -implement connection pools, and pub/sub patterns. - - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Queue` - - A FIFO queue. - - * - :class:`PriorityQueue` - - A priority queue. - - * - :class:`LifoQueue` - - A LIFO queue. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Queue to distribute workload between several - Tasks `. - -* See also the :ref:`Queues documentation page `. - - -Subprocesses -============ - -Utilities to spawn subprocesses and run shell commands. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`create_subprocess_exec` - - Create a subprocess. - - * - ``await`` :func:`create_subprocess_shell` - - Run a shell command. - - -.. rubric:: Examples - -* :ref:`Executing a shell command `. - -* See also the :ref:`subprocess APIs ` - documentation. - - -Streams -======= - -High-level APIs to work with network IO. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`open_connection` - - Establish a TCP connection. - - * - ``await`` :func:`open_unix_connection` - - Establish a Unix socket connection. - - * - ``await`` :func:`start_server` - - Start a TCP server. - - * - ``await`` :func:`start_unix_server` - - Start a Unix socket server. - - * - :class:`StreamReader` - - High-level async/await object to receive network data. - - * - :class:`StreamWriter` - - High-level async/await object to send network data. - - -.. rubric:: Examples - -* :ref:`Example TCP client `. - -* See also the :ref:`streams APIs ` - documentation. - - -Synchronization -=============== - -Threading-like synchronization primitives that can be used in Tasks. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Lock` - - A mutex lock. - - * - :class:`Event` - - An event object. - - * - :class:`Condition` - - A condition object. - - * - :class:`Semaphore` - - A semaphore. - - * - :class:`BoundedSemaphore` - - A bounded semaphore. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Event `. - -* See also the documentation of asyncio - :ref:`synchronization primitives `. - - -Exceptions -========== - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - - * - :exc:`asyncio.TimeoutError` - - Raised on timeout by functions like :func:`wait_for`. - Keep in mind that ``asyncio.TimeoutError`` is **unrelated** - to the built-in :exc:`TimeoutError` exception. - - * - :exc:`asyncio.CancelledError` - - Raised when a Task is cancelled. See also :meth:`Task.cancel`. - - -.. rubric:: Examples - -* :ref:`Handling CancelledError to run code on cancellation request - `. - -* See also the full list of - :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/collections.rst.bak b/Doc/library/collections.rst.bak deleted file mode 100644 index aa0acfaae45a5f0..000000000000000 --- a/Doc/library/collections.rst.bak +++ /dev/null @@ -1,1278 +0,0 @@ -:mod:`collections` --- Container datatypes -========================================== - -.. module:: collections - :synopsis: Container datatypes - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/collections/__init__.py` - -.. testsetup:: * - - from collections import * - import itertools - __name__ = '' - --------------- - -This module implements specialized container datatypes providing alternatives to -Python's general purpose built-in containers, :class:`dict`, :class:`list`, -:class:`set`, and :class:`tuple`. - -===================== ==================================================================== -:func:`namedtuple` factory function for creating tuple subclasses with named fields -:class:`deque` list-like container with fast appends and pops on either end -:class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting hashable objects -:class:`OrderedDict` dict subclass that remembers the order entries were added -:class:`defaultdict` dict subclass that calls a factory function to supply missing values -:class:`UserDict` wrapper around dictionary objects for easier dict subclassing -:class:`UserList` wrapper around list objects for easier list subclassing -:class:`UserString` wrapper around string objects for easier string subclassing -===================== ==================================================================== - - -:class:`ChainMap` objects -------------------------- - -.. versionadded:: 3.3 - -A :class:`ChainMap` class is provided for quickly linking a number of mappings -so they can be treated as a single unit. It is often much faster than creating -a new dictionary and running multiple :meth:`~dict.update` calls. - -The class can be used to simulate nested scopes and is useful in templating. - -.. class:: ChainMap(*maps) - - A :class:`ChainMap` groups multiple dicts or other mappings together to - create a single, updateable view. If no *maps* are specified, a single empty - dictionary is provided so that a new chain always has at least one mapping. - - The underlying mappings are stored in a list. That list is public and can - be accessed or updated using the *maps* attribute. There is no other state. - - Lookups search the underlying mappings successively until a key is found. In - contrast, writes, updates, and deletions only operate on the first mapping. - - A :class:`ChainMap` incorporates the underlying mappings by reference. So, if - one of the underlying mappings gets updated, those changes will be reflected - in :class:`ChainMap`. - - All of the usual dictionary methods are supported. In addition, there is a - *maps* attribute, a method for creating new subcontexts, and a property for - accessing all but the first mapping: - - .. attribute:: maps - - A user updateable list of mappings. The list is ordered from - first-searched to last-searched. It is the only stored state and can - be modified to change which mappings are searched. The list should - always contain at least one mapping. - - .. method:: new_child(m=None) - - Returns a new :class:`ChainMap` containing a new map followed by - all of the maps in the current instance. If ``m`` is specified, - it becomes the new map at the front of the list of mappings; if not - specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for - creating subcontexts that can be updated without altering values in any - of the parent mappings. - - .. versionchanged:: 3.4 - The optional ``m`` parameter was added. - - .. attribute:: parents - - Property returning a new :class:`ChainMap` containing all of the maps in - the current instance except the first one. This is useful for skipping - the first map in the search. Use cases are similar to those for the - :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in - :func:`super` function. A reference to ``d.parents`` is equivalent to: - ``ChainMap(*d.maps[1:])``. - - Note, the iteration order of a :class:`ChainMap()` is determined by - scanning the mappings last to first:: - - >>> baseline = {'music': 'bach', 'art': 'rembrandt'} - >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} - >>> list(ChainMap(adjustments, baseline)) - ['music', 'art', 'opera'] - - This gives the same ordering as a series of :meth:`dict.update` calls - starting with the last mapping:: - - >>> combined = baseline.copy() - >>> combined.update(adjustments) - >>> list(combined) - ['music', 'art', 'opera'] - - .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. - -.. seealso:: - - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. - - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :attr:`~collections.ChainMap.parents` property. - - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. - - * A `greatly simplified read-only version of Chainmap - `_. - - -:class:`ChainMap` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with chained maps. - - -Example of simulating Python's internal lookup chain:: - - import builtins - pylookup = ChainMap(locals(), globals(), vars(builtins)) - -Example of letting user specified command-line arguments take precedence over -environment variables which in turn take precedence over default values:: - - import os, argparse - - defaults = {'color': 'red', 'user': 'guest'} - - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user') - parser.add_argument('-c', '--color') - namespace = parser.parse_args() - command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} - - combined = ChainMap(command_line_args, os.environ, defaults) - print(combined['color']) - print(combined['user']) - -Example patterns for using the :class:`ChainMap` class to simulate nested -contexts:: - - c = ChainMap() # Create root context - d = c.new_child() # Create nested child context - e = c.new_child() # Child of c, independent from d - e.maps[0] # Current context dictionary -- like Python's locals() - e.maps[-1] # Root context -- like Python's globals() - e.parents # Enclosing context chain -- like Python's nonlocals - - d['x'] = 1 # Set value in current context - d['x'] # Get first key in the chain of contexts - del d['x'] # Delete from current context - list(d) # All nested values - k in d # Check all nested values - len(d) # Number of nested values - d.items() # All nested items - dict(d) # Flatten into a regular dictionary - -The :class:`ChainMap` class only makes updates (writes and deletions) to the -first mapping in the chain while lookups will search the full chain. However, -if deep writes and deletions are desired, it is easy to make a subclass that -updates keys found deeper in the chain:: - - class DeepChainMap(ChainMap): - 'Variant of ChainMap that allows direct updates to inner scopes' - - def __setitem__(self, key, value): - for mapping in self.maps: - if key in mapping: - mapping[key] = value - return - self.maps[0][key] = value - - def __delitem__(self, key): - for mapping in self.maps: - if key in mapping: - del mapping[key] - return - raise KeyError(key) - - >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) - >>> d['lion'] = 'orange' # update an existing key two levels down - >>> d['snake'] = 'red' # new keys get added to the topmost dict - >>> del d['elephant'] # remove an existing key one level down - >>> d # display result - DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) - - -:class:`Counter` objects ------------------------- - -A counter tool is provided to support convenient and rapid tallies. -For example:: - - >>> # Tally occurrences of words in a list - >>> cnt = Counter() - >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: - ... cnt[word] += 1 - >>> cnt - Counter({'blue': 3, 'red': 2, 'green': 1}) - - >>> # Find the ten most common words in Hamlet - >>> import re - >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) - >>> Counter(words).most_common(10) - [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), - ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] - -.. class:: Counter([iterable-or-mapping]) - - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. - It is a collection where elements are stored as dictionary keys - and their counts are stored as dictionary values. Counts are allowed to be - any integer value including zero or negative counts. The :class:`Counter` - class is similar to bags or multisets in other languages. - - Elements are counted from an *iterable* or initialized from another - *mapping* (or counter): - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping - >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args - - Counter objects have a dictionary interface except that they return a zero - count for missing items instead of raising a :exc:`KeyError`: - - >>> c = Counter(['eggs', 'ham']) - >>> c['bacon'] # count of a missing element is zero - 0 - - Setting a count to zero does not remove an element from a counter. - Use ``del`` to remove it entirely: - - >>> c['sausage'] = 0 # counter entry with a zero count - >>> del c['sausage'] # del actually removes the entry - - .. versionadded:: 3.1 - - .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` - Inherited the capability to remember insertion order. Math operations - on *Counter* objects also preserve order. Results are ordered - according to when an element is first encountered in the left operand - and then by the order encountered in the right operand. - - Counter objects support three methods beyond those available for all - dictionaries: - - .. method:: elements() - - Return an iterator over elements repeating each as many times as its - count. Elements are returned in the order first encountered. If an - element's count is less than one, :meth:`elements` will ignore it. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> sorted(c.elements()) - ['a', 'a', 'a', 'a', 'b', 'b'] - - .. method:: most_common([n]) - - Return a list of the *n* most common elements and their counts from the - most common to the least. If *n* is omitted or ``None``, - :meth:`most_common` returns *all* elements in the counter. - Elements with equal counts are ordered in the order first encountered: - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('b', 2), ('r', 2)] - - .. method:: subtract([iterable-or-mapping]) - - Elements are subtracted from an *iterable* or from another *mapping* - (or counter). Like :meth:`dict.update` but subtracts counts instead - of replacing them. Both inputs and outputs may be zero or negative. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> d = Counter(a=1, b=2, c=3, d=4) - >>> c.subtract(d) - >>> c - Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) - - .. versionadded:: 3.2 - - The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. - - .. method:: fromkeys(iterable) - - This class method is not implemented for :class:`Counter` objects. - - .. method:: update([iterable-or-mapping]) - - Elements are counted from an *iterable* or added-in from another - *mapping* (or counter). Like :meth:`dict.update` but adds counts - instead of replacing them. Also, the *iterable* is expected to be a - sequence of elements, not a sequence of ``(key, value)`` pairs. - -Counters support rich comparison operators for equality, subset, and -superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. -All of those tests treat missing elements as having zero counts so that -``Counter(a=1) == Counter(a=1, b=0)`` returns true. - -.. versionadded:: 3.10 - Rich comparison operations we were added - -.. versionchanged:: 3.10 - In equality tests, missing elements are treated as having zero counts. - Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered - distinct. - -Common patterns for working with :class:`Counter` objects:: - - sum(c.values()) # total of all counts - c.clear() # reset all counts - list(c) # list unique elements - set(c) # convert to a set - dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs - Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n-1:-1] # n least common elements - +c # remove zero and negative counts - -Several mathematical operations are provided for combining :class:`Counter` -objects to produce multisets (counters that have counts greater than zero). -Addition and subtraction combine counters by adding or subtracting the counts -of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. - - >>> c = Counter(a=3, b=1) - >>> d = Counter(a=1, b=2) - >>> c + d # add two counters together: c[x] + d[x] - Counter({'a': 4, 'b': 3}) - >>> c - d # subtract (keeping only positive counts) - Counter({'a': 2}) - >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP - Counter({'a': 1, 'b': 1}) - >>> c | d # union: max(c[x], d[x]) - Counter({'a': 3, 'b': 2}) - -Unary addition and subtraction are shortcuts for adding an empty counter -or subtracting from an empty counter. - - >>> c = Counter(a=2, b=-4) - >>> +c - Counter({'a': 2}) - >>> -c - Counter({'b': 4}) - -.. versionadded:: 3.3 - Added support for unary plus, unary minus, and in-place multiset operations. - -.. note:: - - Counters were primarily designed to work with positive integers to represent - running counts; however, care was taken to not unnecessarily preclude use - cases needing other types or negative values. To help with those use cases, - this section documents the minimum range and type restrictions. - - * The :class:`Counter` class itself is a dictionary subclass with no - restrictions on its keys and values. The values are intended to be numbers - representing counts, but you *could* store anything in the value field. - - * The :meth:`~Counter.most_common` method requires only that the values be orderable. - - * For in-place operations such as ``c[key] += 1``, the value type need only - support addition and subtraction. So fractions, floats, and decimals would - work and negative values are supported. The same is also true for - :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values - for both inputs and outputs. - - * The multiset methods are designed only for use cases with positive values. - The inputs may be negative or zero, but only outputs with positive values - are created. There are no type restrictions, but the value type needs to - support addition, subtraction, and comparison. - - * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and - negative counts. - -.. seealso:: - - * `Bag class `_ - in Smalltalk. - - * Wikipedia entry for `Multisets `_. - - * `C++ multisets `_ - tutorial with examples. - - * For mathematical operations on multisets and their use cases, see - *Knuth, Donald. The Art of Computer Programming Volume II, - Section 4.6.3, Exercise 19*. - - * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`:: - - map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC - - -:class:`deque` objects ----------------------- - -.. class:: deque([iterable, [maxlen]]) - - Returns a new deque object initialized left-to-right (using :meth:`append`) with - data from *iterable*. If *iterable* is not specified, the new deque is empty. - - Deques are a generalization of stacks and queues (the name is pronounced "deck" - and is short for "double-ended queue"). Deques support thread-safe, memory - efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. - - Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for - ``pop(0)`` and ``insert(0, v)`` operations which change both the size and - position of the underlying data representation. - - - If *maxlen* is not specified or is ``None``, deques may grow to an - arbitrary length. Otherwise, the deque is bounded to the specified maximum - length. Once a bounded length deque is full, when new items are added, a - corresponding number of items are discarded from the opposite end. Bounded - length deques provide functionality similar to the ``tail`` filter in - Unix. They are also useful for tracking transactions and other pools of data - where only the most recent activity is of interest. - - - Deque objects support the following methods: - - .. method:: append(x) - - Add *x* to the right side of the deque. - - - .. method:: appendleft(x) - - Add *x* to the left side of the deque. - - - .. method:: clear() - - Remove all elements from the deque leaving it with length 0. - - - .. method:: copy() - - Create a shallow copy of the deque. - - .. versionadded:: 3.5 - - - .. method:: count(x) - - Count the number of deque elements equal to *x*. - - .. versionadded:: 3.2 - - - .. method:: extend(iterable) - - Extend the right side of the deque by appending elements from the iterable - argument. - - - .. method:: extendleft(iterable) - - Extend the left side of the deque by appending elements from *iterable*. - Note, the series of left appends results in reversing the order of - elements in the iterable argument. - - - .. method:: index(x[, start[, stop]]) - - Return the position of *x* in the deque (at or after index *start* - and before index *stop*). Returns the first match or raises - :exc:`ValueError` if not found. - - .. versionadded:: 3.5 - - - .. method:: insert(i, x) - - Insert *x* into the deque at position *i*. - - If the insertion would cause a bounded deque to grow beyond *maxlen*, - an :exc:`IndexError` is raised. - - .. versionadded:: 3.5 - - - .. method:: pop() - - Remove and return an element from the right side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: popleft() - - Remove and return an element from the left side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: remove(value) - - Remove the first occurrence of *value*. If not found, raises a - :exc:`ValueError`. - - - .. method:: reverse() - - Reverse the elements of the deque in-place and then return ``None``. - - .. versionadded:: 3.2 - - - .. method:: rotate(n=1) - - Rotate the deque *n* steps to the right. If *n* is negative, rotate - to the left. - - When the deque is not empty, rotating one step to the right is equivalent - to ``d.appendleft(d.pop())``, and rotating one step to the left is - equivalent to ``d.append(d.popleft())``. - - - Deque objects also provide one read-only attribute: - - .. attribute:: maxlen - - Maximum size of a deque or ``None`` if unbounded. - - .. versionadded:: 3.1 - - -In addition to the above, deques support iteration, pickling, ``len(d)``, -``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with -the :keyword:`in` operator, and subscript references such as ``d[0]`` to access -the first element. Indexed access is O(1) at both ends but slows to O(n) in -the middle. For fast random access, use lists instead. - -Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, -and ``__imul__()``. - -Example: - -.. doctest:: - - >>> from collections import deque - >>> d = deque('ghi') # make a new deque with three items - >>> for elem in d: # iterate over the deque's elements - ... print(elem.upper()) - G - H - I - - >>> d.append('j') # add a new entry to the right side - >>> d.appendleft('f') # add a new entry to the left side - >>> d # show the representation of the deque - deque(['f', 'g', 'h', 'i', 'j']) - - >>> d.pop() # return and remove the rightmost item - 'j' - >>> d.popleft() # return and remove the leftmost item - 'f' - >>> list(d) # list the contents of the deque - ['g', 'h', 'i'] - >>> d[0] # peek at leftmost item - 'g' - >>> d[-1] # peek at rightmost item - 'i' - - >>> list(reversed(d)) # list the contents of a deque in reverse - ['i', 'h', 'g'] - >>> 'h' in d # search the deque - True - >>> d.extend('jkl') # add multiple elements at once - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> d.rotate(1) # right rotation - >>> d - deque(['l', 'g', 'h', 'i', 'j', 'k']) - >>> d.rotate(-1) # left rotation - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - - >>> deque(reversed(d)) # make a new deque in reverse order - deque(['l', 'k', 'j', 'i', 'h', 'g']) - >>> d.clear() # empty the deque - >>> d.pop() # cannot pop from an empty deque - Traceback (most recent call last): - File "", line 1, in -toplevel- - d.pop() - IndexError: pop from an empty deque - - >>> d.extendleft('abc') # extendleft() reverses the input order - >>> d - deque(['c', 'b', 'a']) - - -:class:`deque` Recipes -^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with deques. - -Bounded length deques provide functionality similar to the ``tail`` filter -in Unix:: - - def tail(filename, n=10): - 'Return the last n lines of a file' - with open(filename) as f: - return deque(f, n) - -Another approach to using deques is to maintain a sequence of recently -added elements by appending to the right and popping to the left:: - - def moving_average(iterable, n=3): - # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 - # http://en.wikipedia.org/wiki/Moving_average - it = iter(iterable) - d = deque(itertools.islice(it, n-1)) - d.appendleft(0) - s = sum(d) - for elem in it: - s += elem - d.popleft() - d.append(elem) - yield s / n - -A `round-robin scheduler -`_ can be implemented with -input iterators stored in a :class:`deque`. Values are yielded from the active -iterator in position zero. If that iterator is exhausted, it can be removed -with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with -the :meth:`~deque.rotate` method:: - - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - iterators = deque(map(iter, iterables)) - while iterators: - try: - while True: - yield next(iterators[0]) - iterators.rotate(-1) - except StopIteration: - # Remove an exhausted iterator. - iterators.popleft() - -The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and -deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the ``rotate()`` method to position elements to be popped:: - - def delete_nth(d, n): - d.rotate(-n) - d.popleft() - d.rotate(n) - -To implement :class:`deque` slicing, use a similar approach applying -:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then -reverse the rotation. -With minor variations on that approach, it is easy to implement Forth style -stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, -``rot``, and ``roll``. - - -:class:`defaultdict` objects ----------------------------- - -.. class:: defaultdict([default_factory[, ...]]) - - Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the - built-in :class:`dict` class. It overrides one method and adds one writable - instance variable. The remaining functionality is the same as for the - :class:`dict` class and is not documented here. - - The first argument provides the initial value for the :attr:`default_factory` - attribute; it defaults to ``None``. All remaining arguments are treated the same - as if they were passed to the :class:`dict` constructor, including keyword - arguments. - - - :class:`defaultdict` objects support the following method in addition to the - standard :class:`dict` operations: - - .. method:: __missing__(key) - - If the :attr:`default_factory` attribute is ``None``, this raises a - :exc:`KeyError` exception with the *key* as argument. - - If :attr:`default_factory` is not ``None``, it is called without arguments - to provide a default value for the given *key*, this value is inserted in - the dictionary for the *key*, and returned. - - If calling :attr:`default_factory` raises an exception this exception is - propagated unchanged. - - This method is called by the :meth:`__getitem__` method of the - :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. - - Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using - :attr:`default_factory`. - - - :class:`defaultdict` objects support the following instance variable: - - - .. attribute:: default_factory - - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. - - .. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in - :pep:`584`. - - -:class:`defaultdict` Examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a -sequence of key-value pairs into a dictionary of lists: - - >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] - >>> d = defaultdict(list) - >>> for k, v in s: - ... d[k].append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` -operation then attaches the value to the new list. When keys are encountered -again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is -simpler and faster than an equivalent technique using :meth:`dict.setdefault`: - - >>> d = {} - >>> for k, v in s: - ... d.setdefault(k, []).append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the -:class:`defaultdict` useful for counting (like a bag or multiset in other -languages): - - >>> s = 'mississippi' - >>> d = defaultdict(int) - >>> for k in s: - ... d[k] += 1 - ... - >>> sorted(d.items()) - [('i', 4), ('m', 1), ('p', 2), ('s', 4)] - -When a letter is first encountered, it is missing from the mapping, so the -:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of -zero. The increment operation then builds up the count for each letter. - -The function :func:`int` which always returns zero is just a special case of -constant functions. A faster and more flexible way to create constant functions -is to use a lambda function which can supply any constant value (not just -zero): - - >>> def constant_factory(value): - ... return lambda: value - >>> d = defaultdict(constant_factory('')) - >>> d.update(name='John', action='ran') - >>> '%(name)s %(action)s to %(object)s' % d - 'John ran to ' - -Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the -:class:`defaultdict` useful for building a dictionary of sets: - - >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] - >>> d = defaultdict(set) - >>> for k, v in s: - ... d[k].add(v) - ... - >>> sorted(d.items()) - [('blue', {2, 4}), ('red', {1, 3})] - - -:func:`namedtuple` Factory Function for Tuples with Named Fields ----------------------------------------------------------------- - -Named tuples assign meaning to each position in a tuple and allow for more readable, -self-documenting code. They can be used wherever regular tuples are used, and -they add the ability to access fields by name instead of position index. - -.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) - - Returns a new tuple subclass named *typename*. The new subclass is used to - create tuple-like objects that have fields accessible by attribute lookup as - well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. - - The *field_names* are a sequence of strings such as ``['x', 'y']``. - Alternatively, *field_names* can be a single string with each fieldname - separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. - - Any valid Python identifier may be used for a fieldname except for names - starting with an underscore. Valid identifiers consist of letters, digits, - and underscores but do not start with a digit or underscore and cannot be - a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, - or *raise*. - - If *rename* is true, invalid fieldnames are automatically replaced - with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is - converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword - ``def`` and the duplicate fieldname ``abc``. - - *defaults* can be ``None`` or an :term:`iterable` of default values. - Since fields with a default value must come after any fields without a - default, the *defaults* are applied to the rightmost parameters. For - example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are - ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to - ``1``, and ``z`` will default to ``2``. - - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. - - Named tuple instances do not have per-instance dictionaries, so they are - lightweight and require no more memory than regular tuples. - - To support pickling, the named tuple class should be assigned to a variable - that matches *typename*. - - .. versionchanged:: 3.1 - Added support for *rename*. - - .. versionchanged:: 3.6 - The *verbose* and *rename* parameters became - :ref:`keyword-only arguments `. - - .. versionchanged:: 3.6 - Added the *module* parameter. - - .. versionchanged:: 3.7 - Removed the *verbose* parameter and the :attr:`_source` attribute. - - .. versionchanged:: 3.7 - Added the *defaults* parameter and the :attr:`_field_defaults` - attribute. - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> # Basic example - >>> Point = namedtuple('Point', ['x', 'y']) - >>> p = Point(11, y=22) # instantiate with positional or keyword arguments - >>> p[0] + p[1] # indexable like the plain tuple (11, 22) - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessible by name - 33 - >>> p # readable __repr__ with a name=value style - Point(x=11, y=22) - -Named tuples are especially useful for assigning field names to result tuples returned -by the :mod:`csv` or :mod:`sqlite3` modules:: - - EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') - - import csv - for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): - print(emp.name, emp.title) - - import sqlite3 - conn = sqlite3.connect('/companydata') - cursor = conn.cursor() - cursor.execute('SELECT name, age, title, department, paygrade FROM employees') - for emp in map(EmployeeRecord._make, cursor.fetchall()): - print(emp.name, emp.title) - -In addition to the methods inherited from tuples, named tuples support -three additional methods and two attributes. To prevent conflicts with -field names, the method and attribute names start with an underscore. - -.. classmethod:: somenamedtuple._make(iterable) - - Class method that makes a new instance from an existing sequence or iterable. - - .. doctest:: - - >>> t = [11, 22] - >>> Point._make(t) - Point(x=11, y=22) - -.. method:: somenamedtuple._asdict() - - Return a new :class:`dict` which maps field names to their corresponding - values: - - .. doctest:: - - >>> p = Point(x=11, y=22) - >>> p._asdict() - {'x': 11, 'y': 22} - - .. versionchanged:: 3.1 - Returns an :class:`OrderedDict` instead of a regular :class:`dict`. - - .. versionchanged:: 3.8 - Returns a regular :class:`dict` instead of an :class:`OrderedDict`. - As of Python 3.7, regular dicts are guaranteed to be ordered. If the - extra features of :class:`OrderedDict` are required, the suggested - remediation is to cast the result to the desired type: - ``OrderedDict(nt._asdict())``. - -.. method:: somenamedtuple._replace(**kwargs) - - Return a new instance of the named tuple replacing specified fields with new - values:: - - >>> p = Point(x=11, y=22) - >>> p._replace(x=33) - Point(x=33, y=22) - - >>> for partnum, record in inventory.items(): - ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) - -.. attribute:: somenamedtuple._fields - - Tuple of strings listing the field names. Useful for introspection - and for creating new named tuple types from existing named tuples. - - .. doctest:: - - >>> p._fields # view the field names - ('x', 'y') - - >>> Color = namedtuple('Color', 'red green blue') - >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) - >>> Pixel(11, 22, 128, 255, 0) - Pixel(x=11, y=22, red=128, green=255, blue=0) - -.. attribute:: somenamedtuple._field_defaults - - Dictionary mapping field names to default values. - - .. doctest:: - - >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) - >>> Account._field_defaults - {'balance': 0} - >>> Account('premium') - Account(type='premium', balance=0) - -To retrieve a field whose name is stored in a string, use the :func:`getattr` -function: - - >>> getattr(p, 'x') - 11 - -To convert a dictionary to a named tuple, use the double-star-operator -(as described in :ref:`tut-unpacking-arguments`): - - >>> d = {'x': 11, 'y': 22} - >>> Point(**d) - Point(x=11, y=22) - -Since a named tuple is a regular Python class, it is easy to add or change -functionality with a subclass. Here is how to add a calculated field and -a fixed-width print format: - -.. doctest:: - - >>> class Point(namedtuple('Point', ['x', 'y'])): - ... __slots__ = () - ... @property - ... def hypot(self): - ... return (self.x ** 2 + self.y ** 2) ** 0.5 - ... def __str__(self): - ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) - - >>> for p in Point(3, 4), Point(14, 5/7): - ... print(p) - Point: x= 3.000 y= 4.000 hypot= 5.000 - Point: x=14.000 y= 0.714 hypot=14.018 - -The subclass shown above sets ``__slots__`` to an empty tuple. This helps -keep memory requirements low by preventing the creation of instance dictionaries. - -Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: - - >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) - -Docstrings can be customized by making direct assignments to the ``__doc__`` -fields: - - >>> Book = namedtuple('Book', ['id', 'title', 'authors']) - >>> Book.__doc__ += ': Hardcover book in active collection' - >>> Book.id.__doc__ = '13-digit ISBN' - >>> Book.title.__doc__ = 'Title of first printing' - >>> Book.authors.__doc__ = 'List of authors sorted by last name' - -.. versionchanged:: 3.5 - Property docstrings became writeable. - -.. seealso:: - - * See :class:`typing.NamedTuple` for a way to add type hints for named - tuples. It also provides an elegant notation using the :keyword:`class` - keyword:: - - class Component(NamedTuple): - part_number: int - weight: float - description: Optional[str] = None - - * See :meth:`types.SimpleNamespace` for a mutable namespace based on an - underlying dictionary instead of a tuple. - - * The :mod:`dataclasses` module provides a decorator and functions for - automatically adding generated special methods to user-defined classes. - - -:class:`OrderedDict` objects ----------------------------- - -Ordered dictionaries are just like regular dictionaries but have some extra -capabilities relating to ordering operations. They have become less -important now that the built-in :class:`dict` class gained the ability -to remember insertion order (this new behavior became guaranteed in -Python 3.7). - -Some differences from :class:`dict` still remain: - -* The regular :class:`dict` was designed to be very good at mapping - operations. Tracking insertion order was secondary. - -* The :class:`OrderedDict` was designed to be good at reordering operations. - Space efficiency, iteration speed, and the performance of update - operations were secondary. - -* Algorithmically, :class:`OrderedDict` can handle frequent reordering - operations better than :class:`dict`. This makes it suitable for tracking - recent accesses (for example in an `LRU cache - `_). - -* The equality operation for :class:`OrderedDict` checks for matching order. - -* The :meth:`popitem` method of :class:`OrderedDict` has a different - signature. It accepts an optional argument to specify which item is popped. - -* :class:`OrderedDict` has a :meth:`move_to_end` method to - efficiently reposition an element to an endpoint. - -* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. - - -.. class:: OrderedDict([items]) - - Return an instance of a :class:`dict` subclass that has methods - specialized for rearranging dictionary order. - - .. versionadded:: 3.1 - - .. method:: popitem(last=True) - - The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in - :abbr:`LIFO (last-in, first-out)` order if *last* is true - or :abbr:`FIFO (first-in, first-out)` order if false. - - .. method:: move_to_end(key, last=True) - - Move an existing *key* to either end of an ordered dictionary. The item - is moved to the right end if *last* is true (the default) or to the - beginning if *last* is false. Raises :exc:`KeyError` if the *key* does - not exist:: - - >>> d = OrderedDict.fromkeys('abcde') - >>> d.move_to_end('b') - >>> ''.join(d.keys()) - 'acdeb' - >>> d.move_to_end('b', last=False) - >>> ''.join(d.keys()) - 'bacde' - - .. versionadded:: 3.2 - -In addition to the usual mapping methods, ordered dictionaries also support -reverse iteration using :func:`reversed`. - -Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. -Equality tests between :class:`OrderedDict` objects and other -:class:`~collections.abc.Mapping` objects are order-insensitive like regular -dictionaries. This allows :class:`OrderedDict` objects to be substituted -anywhere a regular dictionary is used. - -.. versionchanged:: 3.5 - The items, keys, and values :term:`views ` - of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. - -.. versionchanged:: 3.6 - With the acceptance of :pep:`468`, order is retained for keyword arguments - passed to the :class:`OrderedDict` constructor and its :meth:`update` - method. - -.. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. - - -:class:`OrderedDict` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is straightforward to create an ordered dictionary variant -that remembers the order the keys were *last* inserted. -If a new entry overwrites an existing entry, the -original insertion position is changed and moved to the end:: - - class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' - - def __setitem__(self, key, value): - super().__setitem__(key, value) - self.move_to_end(key) - -An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`:: - - class LRU(OrderedDict): - 'Limit size, evicting the least recently looked-up key when full' - - def __init__(self, maxsize=128, /, *args, **kwds): - self.maxsize = maxsize - super().__init__(*args, **kwds) - - def __getitem__(self, key): - value = super().__getitem__(key) - self.move_to_end(key) - return value - - def __setitem__(self, key, value): - if key in self: - self.move_to_end(key) - super().__setitem__(key, value) - if len(self) > self.maxsize: - oldest = next(iter(self)) - del self[oldest] - - -:class:`UserDict` objects -------------------------- - -The class, :class:`UserDict` acts as a wrapper around dictionary objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`dict`; however, this class can be easier -to work with because the underlying dictionary is accessible as an -attribute. - -.. class:: UserDict([initialdata]) - - Class that simulates a dictionary. The instance's contents are kept in a - regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it be used for other purposes. - - In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: - - .. attribute:: data - - A real dictionary used to store the contents of the :class:`UserDict` - class. - - - -:class:`UserList` objects -------------------------- - -This class acts as a wrapper around list objects. It is a useful base class -for your own list-like classes which can inherit from them and override -existing methods or add new ones. In this way, one can add new behaviors to -lists. - -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`list`; however, this class can be easier -to work with because the underlying list is accessible as an attribute. - -.. class:: UserList([list]) - - Class that simulates a list. The instance's contents are kept in a regular - list, which is accessible via the :attr:`data` attribute of :class:`UserList` - instances. The instance's contents are initially set to a copy of *list*, - defaulting to the empty list ``[]``. *list* can be any iterable, for - example a real Python list or a :class:`UserList` object. - - In addition to supporting the methods and operations of mutable sequences, - :class:`UserList` instances provide the following attribute: - - .. attribute:: data - - A real :class:`list` object used to store the contents of the - :class:`UserList` class. - -**Subclassing requirements:** Subclasses of :class:`UserList` are expected to -offer a constructor which can be called with either no arguments or one -argument. List operations which return a new sequence attempt to create an -instance of the actual implementation class. To do so, it assumes that the -constructor can be called with a single parameter, which is a sequence object -used as a data source. - -If a derived class does not wish to comply with this requirement, all of the -special methods supported by this class will need to be overridden; please -consult the sources for information about the methods which need to be provided -in that case. - -:class:`UserString` objects ---------------------------- - -The class, :class:`UserString` acts as a wrapper around string objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`str`; however, this class can be easier -to work with because the underlying string is accessible as an -attribute. - -.. class:: UserString(seq) - - Class that simulates a string object. The instance's - content is kept in a regular string object, which is accessible via the - :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *seq*. The *seq* argument can - be any object which can be converted into a string using the built-in - :func:`str` function. - - In addition to supporting the methods and operations of strings, - :class:`UserString` instances provide the following attribute: - - .. attribute:: data - - A real :class:`str` object used to store the contents of the - :class:`UserString` class. - - .. versionchanged:: 3.5 - New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, - ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/dataclasses.rst.bak b/Doc/library/dataclasses.rst.bak deleted file mode 100644 index e706f7fcc566d89..000000000000000 --- a/Doc/library/dataclasses.rst.bak +++ /dev/null @@ -1,595 +0,0 @@ -:mod:`dataclasses` --- Data Classes -=================================== - -.. module:: dataclasses - :synopsis: Generate special methods on user-defined classes. - -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - -**Source code:** :source:`Lib/dataclasses.py` - --------------- - -This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`__init__` and -:meth:`__repr__` to user-defined classes. It was originally described -in :pep:`557`. - -The member variables to use in these generated methods are defined -using :pep:`526` type annotations. For example this code:: - - from dataclasses import dataclass - - @dataclass - class InventoryItem: - """Class for keeping track of an item in inventory.""" - name: str - unit_price: float - quantity_on_hand: int = 0 - - def total_cost(self) -> float: - return self.unit_price * self.quantity_on_hand - -Will add, among other things, a :meth:`__init__` that looks like:: - - def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): - self.name = name - self.unit_price = unit_price - self.quantity_on_hand = quantity_on_hand - -Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. - -.. versionadded:: 3.7 - -Module-level decorators, classes, and functions ------------------------------------------------ - -.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - - This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. - - The :func:`dataclass` decorator examines the class to find - ``field``\s. A ``field`` is defined as class variable that has a - :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` - examines the type specified in the variable annotation. - - The order of the fields in all of the generated methods is the - order in which they appear in the class definition. - - The :func:`dataclass` decorator will add various "dunder" methods to - the class, described below. If any of the added methods already - exist on the class, the behavior depends on the parameter, as documented - below. The decorator returns the same class that is called on; no new - class is created. - - If :func:`dataclass` is used just as a simple decorator with no parameters, - it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are - equivalent:: - - @dataclass - class C: - ... - - @dataclass() - class C: - ... - - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - class C: - ... - - The parameters to :func:`dataclass` are: - - - ``init``: If true (the default), a :meth:`__init__` method will be - generated. - - If the class already defines :meth:`__init__`, this parameter is - ignored. - - - ``repr``: If true (the default), a :meth:`__repr__` method will be - generated. The generated repr string will have the class name and - the name and repr of each field, in the order they are defined in - the class. Fields that are marked as being excluded from the repr - are not included. For example: - ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - - If the class already defines :meth:`__repr__`, this parameter is - ignored. - - - ``eq``: If true (the default), an :meth:`__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. - - If the class already defines :meth:`__eq__`, this parameter is - ignored. - - - ``order``: If true (the default is ``False``), :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be - generated. These compare the class as if it were a tuple of its - fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a - :exc:`ValueError` is raised. - - If the class already defines any of :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then - :exc:`TypeError` is raised. - - - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. - - :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are - added to hashed collections such as dictionaries and sets. Having a - :meth:`__hash__` implies that instances of the class are immutable. - Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. - - By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` - method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`__hash__` method. Setting the class - attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`__hash__` documentation. - - If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. - This is a specialized use case and should be considered carefully. - - Here are the rules governing implicit creation of a :meth:`__hash__` - method. Note that you cannot both have an explicit :meth:`__hash__` - method in your dataclass and set ``unsafe_hash=True``; this will result - in a :exc:`TypeError`. - - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` - method of the superclass will be used (if the superclass is - :class:`object`, this means it will fall back to id-based hashing). - - - ``frozen``: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. - - ``field``\s may optionally specify a default value, using normal - Python syntax:: - - @dataclass - class C: - a: int # 'a' has no default value - b: int = 0 # assign a default value for 'b' - - In this example, both ``a`` and ``b`` will be included in the added - :meth:`__init__` method, which will be defined as:: - - def __init__(self, a: int, b: int = 0): - - :exc:`TypeError` will be raised if a field without a default value - follows a field with a default value. This is true either when this - occurs in a single class, or as a result of class inheritance. - -.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) - - For common and simple use cases, no other functionality is - required. There are, however, some dataclass features that - require additional per-field information. To satisfy this need for - additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: - - @dataclass - class C: - mylist: list[int] = field(default_factory=list) - - c = C() - c.mylist += [1, 2, 3] - - As shown above, the ``MISSING`` value is a sentinel object used to - detect if the ``default`` and ``default_factory`` parameters are - provided. This sentinel is used because ``None`` is a valid value - for ``default``. No code should directly use the ``MISSING`` - value. - - The parameters to :func:`field` are: - - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself - replaces the normal position of the default value. - - - ``default_factory``: If provided, it must be a zero-argument - callable that will be called when a default value is needed for - this field. Among other purposes, this can be used to specify - fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. - - - ``init``: If true (the default), this field is included as a - parameter to the generated :meth:`__init__` method. - - - ``repr``: If true (the default), this field is included in the - string returned by the generated :meth:`__repr__` method. - - - ``compare``: If true (the default), this field is included in the - generated equality and comparison methods (:meth:`__eq__`, - :meth:`__gt__`, et al.). - - - ``hash``: This can be a bool or ``None``. If true, this field is - included in the generated :meth:`__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be - the expected behavior. A field should be considered in the hash - if it's used for comparisons. Setting this value to anything - other than ``None`` is discouraged. - - One possible reason to set ``hash=False`` but ``compare=True`` - would be if a field is expensive to compute a hash value for, - that field is needed for equality testing, and there are other - fields that contribute to the type's hash value. Even if a field - is excluded from the hash, it will still be used for comparisons. - - - ``metadata``: This can be a mapping or None. None is treated as - an empty dict. This value is wrapped in - :func:`~types.MappingProxyType` to make it read-only, and exposed - on the :class:`Field` object. It is not used at all by Data - Classes, and is provided as a third-party extension mechanism. - Multiple third-parties can each have their own key, to use as a - namespace in the metadata. - - If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is - provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class - attributes will all contain the default values for the fields, just - as if the default value itself were specified. For example, - after:: - - @dataclass - class C: - x: int - y: int = field(repr=False) - z: int = field(repr=False, default=10) - t: int = 20 - - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. - -.. class:: Field - - :class:`Field` objects describe each defined field. These objects - are created internally, and are returned by the :func:`fields` - module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: - - - ``name``: The name of the field. - - - ``type``: The type of the field. - - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, and ``metadata`` have the identical meaning and - values as they do in the :func:`field` declaration. - - Other attributes may exist, but they are private and must not be - inspected or relied on. - -.. function:: fields(class_or_instance) - - Returns a tuple of :class:`Field` objects that define the fields for this - dataclass. Accepts either a dataclass, or an instance of a dataclass. - Raises :exc:`TypeError` if not passed a dataclass or instance of one. - Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. - -.. function:: asdict(instance, *, dict_factory=dict) - - Converts the dataclass ``instance`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted - to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. For example:: - - @dataclass - class Point: - x: int - y: int - - @dataclass - class C: - mylist: list[Point] - - p = Point(10, 20) - assert asdict(p) == {'x': 10, 'y': 20} - - c = C([Point(0, 0), Point(10, 4)]) - assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} - - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. - -.. function:: astuple(instance, *, tuple_factory=tuple) - - Converts the dataclass ``instance`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. - - Continuing from the previous example:: - - assert astuple(p) == (10, 20) - assert astuple(c) == ([(0, 0), (10, 4)],) - - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. - -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an - iterable whose elements are each either ``name``, ``(name, type)``, - or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have - the same meaning as they do in :func:`dataclass`. - - This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to - a dataclass. This function is provided as a convenience. For - example:: - - C = make_dataclass('C', - [('x', int), - 'y', - ('z', int, field(default=5))], - namespace={'add_one': lambda self: self.x + 1}) - - Is equivalent to:: - - @dataclass - class C: - x: int - y: 'typing.Any' - z: int = 5 - - def add_one(self): - return self.x + 1 - -.. function:: replace(instance, /, **changes) - - Creates a new object of the same type of ``instance``, replacing - fields with values from ``changes``. If ``instance`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. - - The newly returned object is created by calling the :meth:`__init__` - method of the dataclass. This ensures that - :meth:`__post_init__`, if present, is also called. - - Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`__init__` and :meth:`__post_init__`. - - It is an error for ``changes`` to contain any fields that are - defined as having ``init=False``. A :exc:`ValueError` will be raised - in this case. - - Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but - rather are initialized in :meth:`__post_init__`, if they're - initialized at all. It is expected that ``init=False`` fields will - be rarely and judiciously used. If they are used, it might be wise - to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance - copying. - -.. function:: is_dataclass(class_or_instance) - - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. - - If you need to know if a class is an instance of a dataclass (and - not a dataclass itself), then add a further check for ``not - isinstance(obj, type)``:: - - def is_dataclass_instance(obj): - return is_dataclass(obj) and not isinstance(obj, type) - -Post-init processing --------------------- - -The generated :meth:`__init__` code will call a method named -:meth:`__post_init__`, if :meth:`__post_init__` is defined on the -class. It will normally be called as ``self.__post_init__()``. -However, if any ``InitVar`` fields are defined, they will also be -passed to :meth:`__post_init__` in the order they were defined in the -class. If no :meth:`__init__` method is generated, then -:meth:`__post_init__` will not automatically be called. - -Among other uses, this allows for initializing field values that -depend on one or more other fields. For example:: - - @dataclass - class C: - a: float - b: float - c: float = field(init=False) - - def __post_init__(self): - self.c = self.a + self.b - -See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how -:func:`replace` handles ``init=False`` fields. - -Class variables ---------------- - -One of two places where :func:`dataclass` actually inspects the type -of a field is to determine if a field is a class variable as defined -in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded -from consideration as a field and is ignored by the dataclass -mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the -module-level :func:`fields` function. - -Init-only variables -------------------- - -The other place where :func:`dataclass` inspects a type annotation is to -determine if a field is an init-only variable. It does this by seeing -if the type of a field is of type ``dataclasses.InitVar``. If a field -is an ``InitVar``, it is considered a pseudo-field called an init-only -field. As it is not a true field, it is not returned by the -module-level :func:`fields` function. Init-only fields are added as -parameters to the generated :meth:`__init__` method, and are passed to -the optional :meth:`__post_init__` method. They are not otherwise used -by dataclasses. - -For example, suppose a field will be initialized from a database, if a -value is not provided when creating the class:: - - @dataclass - class C: - i: int - j: int = None - database: InitVar[DatabaseType] = None - - def __post_init__(self, database): - if self.j is None and database is not None: - self.j = database.lookup('j') - - c = C(10, database=my_database) - -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. - -Frozen instances ----------------- - -It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can -emulate immutability. In that case, dataclasses will add -:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These -methods will raise a :exc:`FrozenInstanceError` when invoked. - -There is a tiny performance penalty when using ``frozen=True``: -:meth:`__init__` cannot use simple assignment to initialize fields, and -must use :meth:`object.__setattr__`. - -Inheritance ------------ - -When the dataclass is being created by the :meth:`dataclass` decorator, -it looks through all of the class's base classes in reverse MRO (that -is, starting at :class:`object`) and, for each dataclass that it finds, -adds the fields from that base class to an ordered mapping of fields. -After all of the base class fields are added, it adds its own fields -to the ordered mapping. All of the generated methods will use this -combined, calculated ordered mapping of fields. Because the fields -are in insertion order, derived classes override base classes. An -example:: - - @dataclass - class Base: - x: Any = 15.0 - y: int = 0 - - @dataclass - class C(Base): - z: int = 10 - x: int = 15 - -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. - -The generated :meth:`__init__` method for ``C`` will look like:: - - def __init__(self, x: int = 15, y: int = 0, z: int = 10): - -Default factory functions -------------------------- - - If a :func:`field` specifies a ``default_factory``, it is called with - zero arguments when a default value for the field is needed. For - example, to create a new instance of a list, use:: - - mylist: list = field(default_factory=list) - - If a field is excluded from :meth:`__init__` (using ``init=False``) - and the field also specifies ``default_factory``, then the default - factory function will always be called from the generated - :meth:`__init__` function. This happens because there is no other - way to give the field an initial value. - -Mutable default values ----------------------- - - Python stores default member variable values in class attributes. - Consider this example, not using dataclasses:: - - class C: - x = [] - def add(self, element): - self.x.append(element) - - o1 = C() - o2 = C() - o1.add(1) - o2.add(2) - assert o1.x == [1, 2] - assert o1.x is o2.x - - Note that the two instances of class ``C`` share the same class - variable ``x``, as expected. - - Using dataclasses, *if* this code was valid:: - - @dataclass - class D: - x: List = [] - def add(self, element): - self.x += element - - it would generate code similar to:: - - class D: - x = [] - def __init__(self, x=x): - self.x = x - def add(self, element): - self.x += element - - assert D().x is D().x - - This has the same issue as the original example using class ``C``. - That is, two instances of class ``D`` that do not specify a value for - ``x`` when creating a class instance will share the same copy of - ``x``. Because dataclasses just use normal Python class creation - they also share this behavior. There is no general way for Data - Classes to detect this condition. Instead, dataclasses will raise a - :exc:`TypeError` if it detects a default parameter of type ``list``, - ``dict``, or ``set``. This is a partial solution, but it does protect - against many common errors. - - Using default factory functions is a way to create new instances of - mutable types as default values for fields:: - - @dataclass - class D: - x: list = field(default_factory=list) - - assert D().x is not D().x - -Exceptions ----------- - -.. exception:: FrozenInstanceError - - Raised when an implicitly defined :meth:`__setattr__` or - :meth:`__delattr__` is called on a dataclass which was defined with - ``frozen=True``. diff --git a/Doc/library/gc.rst.bak b/Doc/library/gc.rst.bak deleted file mode 100644 index a3d201d5055c8ec..000000000000000 --- a/Doc/library/gc.rst.bak +++ /dev/null @@ -1,322 +0,0 @@ -:mod:`gc` --- Garbage Collector interface -========================================= - -.. module:: gc - :synopsis: Interface to the cycle-detecting garbage collector. - -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - --------------- - -This module provides an interface to the optional garbage collector. It -provides the ability to disable the collector, tune the collection frequency, -and set debugging options. It also provides access to unreachable objects that -the collector found but cannot free. Since the collector supplements the -reference counting already used in Python, you can disable the collector if you -are sure your program does not create reference cycles. Automatic collection -can be disabled by calling ``gc.disable()``. To debug a leaking program call -``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes -``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in -gc.garbage for inspection. - -The :mod:`gc` module provides the following functions: - - -.. function:: enable() - - Enable automatic garbage collection. - - -.. function:: disable() - - Disable automatic garbage collection. - - -.. function:: isenabled() - - Return ``True`` if automatic collection is enabled. - - -.. function:: collect(generation=2) - - With no arguments, run a full collection. The optional argument *generation* - may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. - - The free lists maintained for a number of built-in types are cleared - whenever a full collection or collection of the highest generation (2) - is run. Not all items in some free lists may be freed due to the - particular implementation, in particular :class:`float`. - - -.. function:: set_debug(flags) - - Set the garbage collection debugging flags. Debugging information will be - written to ``sys.stderr``. See below for a list of debugging flags which can be - combined using bit operations to control debugging. - - -.. function:: get_debug() - - Return the debugging flags currently set. - - -.. function:: get_objects(generation=None) - - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by - the collector that are in that generation. - - .. versionchanged:: 3.8 - New *generation* parameter. - -.. function:: get_stats() - - Return a list of three per-generation dictionaries containing collection - statistics since interpreter start. The number of keys may change - in the future, but currently each dictionary will contain the following - items: - - * ``collections`` is the number of times this generation was collected; - - * ``collected`` is the total number of objects collected inside this - generation; - - * ``uncollectable`` is the total number of objects which were found - to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. - - .. versionadded:: 3.4 - - -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) - - Set the garbage collection thresholds (the collection frequency). Setting - *threshold0* to zero disables collection. - - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object - allocations and deallocations since the last collection. When the number of - allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. - With the third generation, things are a bit more complicated, - see `Collecting the oldest generation `_ for more information. - - -.. function:: get_count() - - Return the current collection counts as a tuple of ``(count0, count1, - count2)``. - - -.. function:: get_threshold() - - Return the current collection thresholds as a tuple of ``(threshold0, - threshold1, threshold2)``. - - -.. function:: get_referrers(*objs) - - Return the list of objects that directly refer to any of objs. This function - will only locate those containers which support garbage collection; extension - types which do refer to other objects but do not support garbage collection will - not be found. - - Note that objects which have already been dereferenced, but which live in cycles - and have not yet been collected by the garbage collector can be listed among the - resulting referrers. To get only currently live objects, call :func:`collect` - before calling :func:`get_referrers`. - - .. warning:: - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. - - -.. function:: get_referents(*objs) - - Return a list of objects directly referred to by any of the arguments. The - referents returned are those objects visited by the arguments' C-level - :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects - that support garbage collection, and are only required to visit objects that may - be involved in a cycle. So, for example, if an integer is directly reachable - from an argument, that integer object may or may not appear in the result list. - - -.. function:: is_tracked(obj) - - Returns ``True`` if the object is currently tracked by the garbage collector, - ``False`` otherwise. As a general rule, instances of atomic types aren't - tracked and instances of non-atomic types (containers, user-defined - objects...) are. However, some type-specific optimizations can be present - in order to suppress the garbage collector footprint of simple instances - (e.g. dicts containing only atomic keys and values):: - - >>> gc.is_tracked(0) - False - >>> gc.is_tracked("a") - False - >>> gc.is_tracked([]) - True - >>> gc.is_tracked({}) - False - >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) - True - - .. versionadded:: 3.1 - - -.. function:: is_finalized(obj) - - Returns ``True`` if the given object has been finalized by the - garbage collector, ``False`` otherwise. :: - - >>> x = None - >>> class Lazarus: - ... def __del__(self): - ... global x - ... x = self - ... - >>> lazarus = Lazarus() - >>> gc.is_finalized(lazarus) - False - >>> del lazarus - >>> gc.is_finalized(x) - True - - .. versionadded:: 3.9 - - -.. function:: freeze() - - Freeze all the objects tracked by gc - move them to a permanent generation - and ignore all the future collections. This can be used before a POSIX - fork() call to make the gc copy-on-write friendly or to speed up collection. - Also collection before a POSIX fork() call may free pages for future - allocation which can cause copy-on-write too so it's advised to disable gc - in parent process and freeze before fork and enable gc in child process. - - .. versionadded:: 3.7 - - -.. function:: unfreeze() - - Unfreeze the objects in the permanent generation, put them back into the - oldest generation. - - .. versionadded:: 3.7 - - -.. function:: get_freeze_count() - - Return the number of objects in the permanent generation. - - .. versionadded:: 3.7 - - -The following variables are provided for read-only access (you can mutate the -values but should not rebind them): - -.. data:: garbage - - A list of objects which the collector found to be unreachable but could - not be freed (uncollectable objects). Starting with Python 3.4, this - list should be empty most of the time, except when using instances of - C extension types with a non-``NULL`` ``tp_del`` slot. - - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be - added to this list rather than freed. - - .. versionchanged:: 3.2 - If this list is non-empty at :term:`interpreter shutdown`, a - :exc:`ResourceWarning` is emitted, which is silent by default. If - :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects - are printed. - - .. versionchanged:: 3.4 - Following :pep:`442`, objects with a :meth:`__del__` method don't end - up in :attr:`gc.garbage` anymore. - -.. data:: callbacks - - A list of callbacks that will be invoked by the garbage collector before and - after collection. The callbacks will be called with two arguments, - *phase* and *info*. - - *phase* can be one of two values: - - "start": The garbage collection is about to start. - - "stop": The garbage collection has finished. - - *info* is a dict providing more information for the callback. The following - keys are currently defined: - - "generation": The oldest generation being collected. - - "collected": When *phase* is "stop", the number of objects - successfully collected. - - "uncollectable": When *phase* is "stop", the number of objects - that could not be collected and were put in :data:`garbage`. - - Applications can add their own callbacks to this list. The primary - use cases are: - - Gathering statistics about garbage collection, such as how often - various generations are collected, and how long the collection - takes. - - Allowing applications to identify and clear their own uncollectable - types when they appear in :data:`garbage`. - - .. versionadded:: 3.3 - - -The following constants are provided for use with :func:`set_debug`: - - -.. data:: DEBUG_STATS - - Print statistics during collection. This information can be useful when tuning - the collection frequency. - - -.. data:: DEBUG_COLLECTABLE - - Print information on collectable objects found. - - -.. data:: DEBUG_UNCOLLECTABLE - - Print information of uncollectable objects found (objects which are not - reachable but cannot be freed by the collector). These objects will be added - to the ``garbage`` list. - - .. versionchanged:: 3.2 - Also print the contents of the :data:`garbage` list at - :term:`interpreter shutdown`, if it isn't empty. - -.. data:: DEBUG_SAVEALL - - When set, all unreachable objects found will be appended to *garbage* rather - than being freed. This can be useful for debugging a leaking program. - - -.. data:: DEBUG_LEAK - - The debugging flags necessary for the collector to print information about a - leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | - DEBUG_SAVEALL``). diff --git a/Doc/library/importlib.metadata.rst.bak b/Doc/library/importlib.metadata.rst.bak deleted file mode 100644 index 7f154ea02cc4f2d..000000000000000 --- a/Doc/library/importlib.metadata.rst.bak +++ /dev/null @@ -1,262 +0,0 @@ -.. _using: - -================================= - Using :mod:`!importlib.metadata` -================================= - -.. note:: - This functionality is provisional and may deviate from the usual - version semantics of the standard library. - -``importlib.metadata`` is a library that provides for access to installed -package metadata. Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources` in Python 3.7 -and newer (backported as `importlib_resources`_ for older versions of -Python), this can eliminate the need to use the older and less efficient -``pkg_resources`` package. - -By "installed package" we generally mean a third-party package installed into -Python's ``site-packages`` directory via tools such as `pip -`_. Specifically, -it means a package with either a discoverable ``dist-info`` or ``egg-info`` -directory, and metadata defined by :pep:`566` or its older specifications. -By default, package metadata can live on the file system or in zip archives on -:data:`sys.path`. Through an extension mechanism, the metadata can live almost -anywhere. - - -Overview -======== - -Let's say you wanted to get the version string for a package you've installed -using ``pip``. We start by creating a virtual environment and installing -something into it: - -.. code-block:: shell-session - - $ python3 -m venv example - $ source example/bin/activate - (example) $ pip install wheel - -You can get the version string for ``wheel`` by running the following: - -.. code-block:: pycon - - (example) $ python - >>> from importlib.metadata import version # doctest: +SKIP - >>> version('wheel') # doctest: +SKIP - '0.32.3' - -You can also get the set of entry points keyed by group, such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a -sequence of :ref:`EntryPoint ` objects. - -You can get the :ref:`metadata for a distribution `:: - - >>> list(metadata('wheel')) # doctest: +SKIP - ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] - -You can also get a :ref:`distribution's version number `, list its -:ref:`constituent files `, and get a list of the distribution's -:ref:`requirements`. - - -Functional API -============== - -This package provides the following functionality via its public API. - - -.. _entry-points: - -Entry points ------------- - -The ``entry_points()`` function returns a dictionary of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; -each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, -``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: - - >>> eps = entry_points() # doctest: +SKIP - >>> list(eps) # doctest: +SKIP - ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] # doctest: +SKIP - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP - >>> wheel # doctest: +SKIP - EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') - >>> wheel.module # doctest: +SKIP - 'wheel.cli' - >>> wheel.attr # doctest: +SKIP - 'main' - >>> wheel.extras # doctest: +SKIP - [] - >>> main = wheel.load() # doctest: +SKIP - >>> main # doctest: +SKIP - - -The ``group`` and ``name`` are arbitrary values defined by the package author -and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs -`_ -for more information on entry points, their definition, and usage. - - -.. _metadata: - -Distribution metadata ---------------------- - -Every distribution includes some metadata, which you can extract using the -``metadata()`` function:: - - >>> wheel_metadata = metadata('wheel') # doctest: +SKIP - -The keys of the returned data structure, a ``PackageMetadata``, -name the metadata keywords, and -the values are returned unparsed from the distribution metadata:: - - >>> wheel_metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - - -.. _version: - -Distribution versions ---------------------- - -The ``version()`` function is the quickest way to get a distribution's version -number, as a string:: - - >>> version('wheel') # doctest: +SKIP - '0.32.3' - - -.. _files: - -Distribution files ------------------- - -You can also get the full set of files contained within a distribution. The -``files()`` function takes a distribution package name and returns all of the -files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: - - >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP - >>> util # doctest: +SKIP - PackagePath('wheel/util.py') - >>> util.size # doctest: +SKIP - 859 - >>> util.dist # doctest: +SKIP - - >>> util.hash # doctest: +SKIP - - -Once you have the file, you can also read its contents:: - - >>> print(util.read_text()) # doctest: +SKIP - import base64 - import sys - ... - def as_bytes(s): - if isinstance(s, text_type): - return s.encode('utf-8') - return s - -In the case where the metadata file listing files -(RECORD or SOURCES.txt) is missing, ``files()`` will -return ``None``. The caller may wish to wrap calls to -``files()`` in `always_iterable -`_ -or otherwise guard against this condition if the target -distribution is not known to have the metadata present. - -.. _requirements: - -Distribution requirements -------------------------- - -To get the full set of requirements for a distribution, use the ``requires()`` -function:: - - >>> requires('wheel') # doctest: +SKIP - ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] - - -Distributions -============= - -While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for a Python package. You can -get the ``Distribution`` instance:: - - >>> from importlib.metadata import distribution # doctest: +SKIP - >>> dist = distribution('wheel') # doctest: +SKIP - -Thus, an alternative way to get the version number is through the -``Distribution`` instance:: - - >>> dist.version # doctest: +SKIP - '0.32.3' - -There are all kinds of additional metadata available on the ``Distribution`` -instance:: - - >>> dist.metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> dist.metadata['License'] # doctest: +SKIP - 'MIT' - -The full set of available metadata is not described here. See :pep:`566` -for additional details. - - -Extending the search algorithm -============================== - -Because package metadata is not available through :data:`sys.path` searches, or -package loaders directly, the metadata for a package is found through import -system :ref:`finders `. To find a distribution package's metadata, -``importlib.metadata`` queries the list of :term:`meta path finders ` on -:data:`sys.meta_path`. - -The default ``PathFinder`` for Python includes a hook that calls into -``importlib.metadata.MetadataPathFinder`` for finding distributions -loaded from typical file-system-based paths. - -The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the -interface expected of finders by Python's import system. -``importlib.metadata`` extends this protocol by looking for an optional -``find_distributions`` callable on the finders from -:data:`sys.meta_path` and presents this extended interface as the -``DistributionFinder`` abstract base class, which defines this abstract -method:: - - @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): - """Return an iterable of all Distribution instances capable of - loading the metadata for packages for the indicated ``context``. - """ - -The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and name to match and may -supply other relevant context. - -What this means in practice is that to support finding distribution package -metadata in locations other than the file system, subclass -``Distribution`` and implement the abstract methods. Then from -a custom finder, return instances of this derived ``Distribution`` in the -``find_distributions()`` method. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api -.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html - - -.. rubric:: Footnotes diff --git a/Doc/library/logging.rst.bak b/Doc/library/logging.rst.bak deleted file mode 100644 index 431a5849fa9bffc..000000000000000 --- a/Doc/library/logging.rst.bak +++ /dev/null @@ -1,1350 +0,0 @@ -:mod:`logging` --- Logging facility for Python -============================================== - -.. module:: logging - :synopsis: Flexible event logging system for applications. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/__init__.py` - -.. index:: pair: Errors; logging - -.. sidebar:: Important - - This page contains the API reference information. For tutorial - information and discussion of more advanced topics, see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This module defines functions and classes which implement a flexible event -logging system for applications and libraries. - -The key benefit of having the logging API provided by a standard library module -is that all Python modules can participate in logging, so your application log -can include your own messages integrated with messages from third-party -modules. - -The module provides a lot of functionality and flexibility. If you are -unfamiliar with logging, the best way to get to grips with it is to see the -tutorials (see the links on the right). - -The basic classes defined by the module, together with their functions, are -listed below. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - - -.. _logger: - -Logger Objects --------------- - -Loggers have the following attributes and methods. Note that Loggers should -*NEVER* be instantiated directly, but always through the module-level function -``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same -name will always return a reference to the same Logger object. - -The ``name`` is potentially a period-separated hierarchical value, like -``foo.bar.baz`` (though it could also be just plain ``foo``, for example). -Loggers that are further down in the hierarchical list are children of loggers -higher up in the list. For example, given a logger with a name of ``foo``, -loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. - - -.. class:: Logger - - .. attribute:: Logger.propagate - - If this attribute evaluates to true, events logged to this logger will be - passed to the handlers of higher level (ancestor) loggers, in addition to - any handlers attached to this logger. Messages are passed directly to the - ancestor loggers' handlers - neither the level nor filters of the ancestor - loggers in question are considered. - - If this evaluates to false, logging messages are not passed to the handlers - of ancestor loggers. - - The constructor sets this attribute to ``True``. - - .. note:: If you attach a handler to a logger *and* one or more of its - ancestors, it may emit the same record multiple times. In general, you - should not need to attach a handler to more than one logger - if you just - attach it to the appropriate logger which is highest in the logger - hierarchy, then it will see all events logged by all descendant loggers, - provided that their propagate setting is left set to ``True``. A common - scenario is to attach handlers only to the root logger, and to let - propagation take care of the rest. - - .. method:: Logger.setLevel(level) - - Sets the threshold for this logger to *level*. Logging messages which are less - severe than *level* will be ignored; logging messages which have severity *level* - or higher will be emitted by whichever handler or handlers service this logger, - unless a handler's level has been set to a higher severity level than *level*. - - When a logger is created, the level is set to :const:`NOTSET` (which causes - all messages to be processed when the logger is the root logger, or delegation - to the parent when the logger is a non-root logger). Note that the root logger - is created with level :const:`WARNING`. - - The term 'delegation to the parent' means that if a logger has a level of - NOTSET, its chain of ancestor loggers is traversed until either an ancestor with - a level other than NOTSET is found, or the root is reached. - - If an ancestor is found with a level other than NOTSET, then that ancestor's - level is treated as the effective level of the logger where the ancestor search - began, and is used to determine how a logging event is handled. - - If the root is reached, and it has a level of NOTSET, then all messages will be - processed. Otherwise, the root's level will be used as the effective level. - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. Note, however, that levels are internally stored - as integers, and methods such as e.g. :meth:`getEffectiveLevel` and - :meth:`isEnabledFor` will return/expect to be passed integers. - - - .. method:: Logger.isEnabledFor(level) - - Indicates if a message of severity *level* would be processed by this logger. - This method checks first the module-level level set by - ``logging.disable(level)`` and then the logger's effective level as determined - by :meth:`getEffectiveLevel`. - - - .. method:: Logger.getEffectiveLevel() - - Indicates the effective level for this logger. If a value other than - :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, - the hierarchy is traversed towards the root until a value other than - :const:`NOTSET` is found, and that value is returned. The value returned is - an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` - etc. - - - .. method:: Logger.getChild(suffix) - - Returns a logger which is a descendant to this logger, as determined by the suffix. - Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same - logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a - convenience method, useful when the parent logger is named using e.g. ``__name__`` - rather than a literal string. - - .. versionadded:: 3.2 - - - .. method:: Logger.debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on this logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - No % formatting operation is performed on *msg* when no *args* are supplied. - - There are four keyword arguments in *kwargs* which are inspected: - *exc_info*, *stack_info*, *stacklevel* and *extra*. - - If *exc_info* does not evaluate as false, it causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *stacklevel*, which defaults to ``1``. - If greater than 1, the corresponding number of stack frames are skipped - when computing the line number and function name set in the :class:`LogRecord` - created for the logging event. This can be used in logging helpers so that - the function name, filename and line number recorded are not the information - for the helper function/method, but rather its caller. The name of this - parameter mirrors the equivalent one in the :mod:`warnings` module. - - The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` - created for the logging event with user-defined attributes. These custom - attributes can then be used as you like. For example, they could be - incorporated into logged messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logger = logging.getLogger('tcpserver') - logger.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the :class:`LogRecord`. If these are missing, the message will - not be logged because a string formatting exception will occur. So in this case, - you always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - - .. versionchanged:: 3.5 - The *exc_info* parameter can now accept exception instances. - - .. versionchanged:: 3.8 - The *stacklevel* parameter was added. - - - .. method:: Logger.info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on this logger. The arguments are - interpreted as for :meth:`debug`. - - .. note:: There is an obsolete method ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - .. method:: Logger.error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.log(level, msg, *args, **kwargs) - - Logs a message with integer level *level* on this logger. The other arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. Exception info is added to the logging - message. This method should only be called from an exception handler. - - - .. method:: Logger.addFilter(filter) - - Adds the specified filter *filter* to this logger. - - - .. method:: Logger.removeFilter(filter) - - Removes the specified filter *filter* from this logger. - - - .. method:: Logger.filter(record) - - Apply this logger's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be processed (passed to handlers). If one returns a false value, no - further processing of the record occurs. - - - .. method:: Logger.addHandler(hdlr) - - Adds the specified handler *hdlr* to this logger. - - - .. method:: Logger.removeHandler(hdlr) - - Removes the specified handler *hdlr* from this logger. - - - .. method:: Logger.findCaller(stack_info=False, stacklevel=1) - - Finds the caller's source filename and line number. Returns the filename, line - number, function name and stack information as a 4-element tuple. The stack - information is returned as ``None`` unless *stack_info* is ``True``. - - The *stacklevel* parameter is passed from code calling the :meth:`debug` - and other APIs. If greater than 1, the excess is used to skip stack frames - before determining the values to be returned. This will generally be useful - when calling logging APIs from helper/wrapper code, so that the information - in the event log refers not to the helper/wrapper code, but to the code that - calls it. - - - .. method:: Logger.handle(record) - - Handles a record by passing it to all handlers associated with this logger and - its ancestors (until a false value of *propagate* is found). This method is used - for unpickled records received from a socket, as well as those created locally. - Logger-level filtering is applied using :meth:`~Logger.filter`. - - - .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) - - This is a factory method which can be overridden in subclasses to create - specialized :class:`LogRecord` instances. - - .. method:: Logger.hasHandlers() - - Checks to see if this logger has any handlers configured. This is done by - looking for handlers in this logger and its parents in the logger hierarchy. - Returns ``True`` if a handler was found, else ``False``. The method stops searching - up the hierarchy whenever a logger with the 'propagate' attribute set to - false is found - that will be the last logger which is checked for the - existence of handlers. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - Loggers can now be pickled and unpickled. - -.. _levels: - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - - -.. _handler: - -Handler Objects ---------------- - -Handlers have the following attributes and methods. Note that :class:`Handler` -is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call -:meth:`Handler.__init__`. - -.. class:: Handler - - .. method:: Handler.__init__(level=NOTSET) - - Initializes the :class:`Handler` instance by setting its level, setting the list - of filters to the empty list and creating a lock (using :meth:`createLock`) for - serializing access to an I/O mechanism. - - - .. method:: Handler.createLock() - - Initializes a thread lock which can be used to serialize access to underlying - I/O functionality which may not be threadsafe. - - - .. method:: Handler.acquire() - - Acquires the thread lock created with :meth:`createLock`. - - - .. method:: Handler.release() - - Releases the thread lock acquired with :meth:`acquire`. - - - .. method:: Handler.setLevel(level) - - Sets the threshold for this handler to *level*. Logging messages which are - less severe than *level* will be ignored. When a handler is created, the - level is set to :const:`NOTSET` (which causes all messages to be - processed). - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. - - - .. method:: Handler.setFormatter(fmt) - - Sets the :class:`Formatter` for this handler to *fmt*. - - - .. method:: Handler.addFilter(filter) - - Adds the specified filter *filter* to this handler. - - - .. method:: Handler.removeFilter(filter) - - Removes the specified filter *filter* from this handler. - - - .. method:: Handler.filter(record) - - Apply this handler's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be emitted. If one returns a false value, the handler will not emit the - record. - - - .. method:: Handler.flush() - - Ensure all logging output has been flushed. This version does nothing and is - intended to be implemented by subclasses. - - - .. method:: Handler.close() - - Tidy up any resources used by the handler. This version does no output but - removes the handler from an internal list of handlers which is closed when - :func:`shutdown` is called. Subclasses should ensure that this gets called - from overridden :meth:`close` methods. - - - .. method:: Handler.handle(record) - - Conditionally emits the specified logging record, depending on filters which may - have been added to the handler. Wraps the actual emission of the record with - acquisition/release of the I/O thread lock. - - - .. method:: Handler.handleError(record) - - This method should be called from handlers when an exception is encountered - during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is - what is mostly wanted for a logging system - most users will not care about - errors in the logging system, they are more interested in application - errors. You could, however, replace this with a custom handler if you wish. - The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is - more useful during development). - - - .. method:: Handler.format(record) - - Do formatting for a record - if a formatter is set, use it. Otherwise, use the - default formatter for the module. - - - .. method:: Handler.emit(record) - - Do whatever it takes to actually log the specified logging record. This version - is intended to be implemented by subclasses and so raises a - :exc:`NotImplementedError`. - -For a list of handlers included as standard, see :mod:`logging.handlers`. - -.. _formatter-objects: - -Formatter Objects ------------------ - -.. currentmodule:: logging - -:class:`Formatter` objects have the following attributes and methods. They are -responsible for converting a :class:`LogRecord` to (usually) a string which can -be interpreted by either a human or an external system. The base -:class:`Formatter` allows a formatting string to be specified. If none is -supplied, the default value of ``'%(message)s'`` is used, which just includes -the message in the logging call. To have additional items of information in the -formatted output (such as a timestamp), keep reading. - -A Formatter can be initialized with a format string which makes use of knowledge -of the :class:`LogRecord` attributes - such as the default value mentioned above -making use of the fact that the user's message and arguments are pre-formatted -into a :class:`LogRecord`'s *message* attribute. This format string contains -standard Python %-style mapping keys. See section :ref:`old-string-formatting` -for more information on string formatting. - -The useful mapping keys in a :class:`LogRecord` are given in the section on -:ref:`logrecord-attributes`. - - -.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) - - Returns a new instance of the :class:`Formatter` class. The instance is - initialized with a format string for the message as a whole, as well as a - format string for the date/time portion of a message. If no *fmt* is - specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format - is used which is described in the :meth:`formatTime` documentation. - - The *style* parameter can be one of '%', '{' or '$' and determines how - the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. This only applies to the - format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the - actual log messages passed to ``Logger.debug`` etc; see - :ref:`formatting-styles` for more information on using {- and $-formatting - for log messages. - - The *defaults* parameter can be a dictionary with default values to use in - custom fields. For example: - ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - - .. versionchanged:: 3.2 - The *style* parameter was added. - - .. versionchanged:: 3.8 - The *validate* parameter was added. Incorrect or mismatched style and fmt - will raise a ``ValueError``. - For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. - - .. versionchanged:: 3.10 - The *defaults* parameter was added. - - .. method:: format(record) - - The record's attribute dictionary is used as the operand to a string - formatting operation. Returns the resulting string. Before formatting the - dictionary, a couple of preparatory steps are carried out. The *message* - attribute of the record is computed using *msg* % *args*. If the - formatting string contains ``'(asctime)'``, :meth:`formatTime` is called - to format the event time. If there is exception information, it is - formatted using :meth:`formatException` and appended to the message. Note - that the formatted exception information is cached in attribute - *exc_text*. This is useful because the exception information can be - pickled and sent across the wire, but you should be careful if you have - more than one :class:`Formatter` subclass which customizes the formatting - of exception information. In this case, you will have to clear the cached - value (by setting the *exc_text* attribute to ``None``) after a formatter - has done its formatting, so that the next formatter to handle the event - doesn't use the cached value, but recalculates it afresh. - - If stack information is available, it's appended after the exception - information, using :meth:`formatStack` to transform it if necessary. - - - .. method:: formatTime(record, datefmt=None) - - This method should be called from :meth:`format` by a formatter which - wants to make use of a formatted time. This method can be overridden in - formatters to provide for any specific requirement, but the basic behavior - is as follows: if *datefmt* (a string) is specified, it is used with - :func:`time.strftime` to format the creation time of the - record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the - uuu part is a millisecond value and the other letters are as per the - :func:`time.strftime` documentation. An example time in this format is - ``2003-01-23 00:29:50,411``. The resulting string is returned. - - This function uses a user-configurable function to convert the creation - time to a tuple. By default, :func:`time.localtime` is used; to change - this for a particular formatter instance, set the ``converter`` attribute - to a function with the same signature as :func:`time.localtime` or - :func:`time.gmtime`. To change it for all formatters, for example if you - want all logging times to be shown in GMT, set the ``converter`` - attribute in the ``Formatter`` class. - - .. versionchanged:: 3.3 - Previously, the default format was hard-coded as in this example: - ``2010-09-06 22:38:15,292`` where the part before the comma is - handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the - part after the comma is a millisecond value. Because strptime does not - have a format placeholder for milliseconds, the millisecond value is - appended using another format string, ``'%s,%03d'`` --- and both of these - format strings have been hardcoded into this method. With the change, - these strings are defined as class-level attributes which can be - overridden at the instance level when desired. The names of the - attributes are ``default_time_format`` (for the strptime format string) - and ``default_msec_format`` (for appending the millisecond value). - - .. versionchanged:: 3.9 - The ``default_msec_format`` can be ``None``. - - .. method:: formatException(exc_info) - - Formats the specified exception information (a standard exception tuple as - returned by :func:`sys.exc_info`) as a string. This default implementation - just uses :func:`traceback.print_exception`. The resulting string is - returned. - - .. method:: formatStack(stack_info) - - Formats the specified stack information (a string as returned by - :func:`traceback.print_stack`, but with the last newline removed) as a - string. This default implementation just returns the input value. - -.. _filter: - -Filter Objects --------------- - -``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated -filtering than is provided by levels. The base filter class only allows events -which are below a certain point in the logger hierarchy. For example, a filter -initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', -'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the -empty string, all events are passed. - - -.. class:: Filter(name='') - - Returns an instance of the :class:`Filter` class. If *name* is specified, it - names a logger which, together with its children, will have its events allowed - through the filter. If *name* is the empty string, allows every event. - - - .. method:: filter(record) - - Is the specified record to be logged? Returns zero for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place by this - method. - -Note that filters attached to handlers are consulted before an event is -emitted by the handler, whereas filters attached to loggers are consulted -whenever an event is logged (using :meth:`debug`, :meth:`info`, -etc.), before sending an event to handlers. This means that events which have -been generated by descendant loggers will not be filtered by a logger's filter -setting, unless the filter has also been applied to those descendant loggers. - -You don't actually need to subclass ``Filter``: you can pass any instance -which has a ``filter`` method with the same semantics. - -.. versionchanged:: 3.2 - You don't need to create specialized ``Filter`` classes, or use other - classes with a ``filter`` method: you can use a function (or other - callable) as a filter. The filtering logic will check to see if the filter - object has a ``filter`` attribute: if it does, it's assumed to be a - ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's - assumed to be a callable and called with the record as the single - parameter. The returned value should conform to that returned by - :meth:`~Filter.filter`. - -Although filters are used primarily to filter records based on more -sophisticated criteria than levels, they get to see every record which is -processed by the handler or logger they're attached to: this can be useful if -you want to do things like counting how many records were processed by a -particular logger or handler, or adding, changing or removing attributes in -the :class:`LogRecord` being processed. Obviously changing the LogRecord needs -to be done with some care, but it does allow the injection of contextual -information into logs (see :ref:`filters-contextual`). - -.. _log-record: - -LogRecord Objects ------------------ - -:class:`LogRecord` instances are created automatically by the :class:`Logger` -every time something is logged, and can be created manually via -:func:`makeLogRecord` (for example, from a pickled event received over the -wire). - - -.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) - - Contains all the information pertinent to the event being logged. - - The primary information is passed in :attr:`msg` and :attr:`args`, which - are combined using ``msg % args`` to create the :attr:`message` field of the - record. - - :param name: The name of the logger used to log the event represented by - this LogRecord. Note that this name will always have this - value, even though it may be emitted by a handler attached to - a different (ancestor) logger. - :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) - Note that this is converted to *two* attributes of the LogRecord: - ``levelno`` for the numeric value and ``levelname`` for the - corresponding level name. - :param pathname: The full pathname of the source file where the logging call - was made. - :param lineno: The line number in the source file where the logging call was - made. - :param msg: The event description message, possibly a format string with - placeholders for variable data. - :param args: Variable data to merge into the *msg* argument to obtain the - event description. - :param exc_info: An exception tuple with the current exception information, - or ``None`` if no exception information is available. - :param func: The name of the function or method from which the logging call - was invoked. - :param sinfo: A text string representing stack information from the base of - the stack in the current thread, up to the logging call. - - .. method:: getMessage() - - Returns the message for this :class:`LogRecord` instance after merging any - user-supplied arguments with the message. If the user-supplied message - argument to the logging call is not a string, :func:`str` is called on it to - convert it to a string. This allows use of user-defined classes as - messages, whose ``__str__`` method can return the actual format string to - be used. - - .. versionchanged:: 3.2 - The creation of a :class:`LogRecord` has been made more configurable by - providing a factory which is used to create the record. The factory can be - set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` - (see this for the factory's signature). - - This functionality can be used to inject your own values into a - :class:`LogRecord` at creation time. You can use the following pattern:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - - With this pattern, multiple factories could be chained, and as long - as they don't overwrite each other's attributes or unintentionally - overwrite the standard attributes listed above, there should be no - surprises. - - -.. _logrecord-attributes: - -LogRecord attributes --------------------- - -The LogRecord has a number of attributes, most of which are derived from the -parameters to the constructor. (Note that the names do not always correspond -exactly between the LogRecord constructor parameters and the LogRecord -attributes.) These attributes can be used to merge data from the record into -the format string. The following table lists (in alphabetical order) the -attribute names, their meanings and the corresponding placeholder in a %-style -format string. - -If you are using {}-formatting (:func:`str.format`), you can use -``{attrname}`` as the placeholder in the format string. If you are using -$-formatting (:class:`string.Template`), use the form ``${attrname}``. In -both cases, of course, replace ``attrname`` with the actual attribute name -you want to use. - -In the case of {}-formatting, you can specify formatting flags by placing them -after the attribute name, separated from it with a colon. For example: a -placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as -``004``. Refer to the :meth:`str.format` documentation for full details on -the options available to you. - -+----------------+-------------------------+-----------------------------------------------+ -| Attribute name | Format | Description | -+================+=========================+===============================================+ -| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | -| | format this yourself. | produce ``message``, or a dict whose values | -| | | are used for the merge (when there is only one| -| | | argument, and it is a dictionary). | -+----------------+-------------------------+-----------------------------------------------+ -| asctime | ``%(asctime)s`` | Human-readable time when the | -| | | :class:`LogRecord` was created. By default | -| | | this is of the form '2003-07-08 16:49:45,896' | -| | | (the numbers after the comma are millisecond | -| | | portion of the time). | -+----------------+-------------------------+-----------------------------------------------+ -| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | -+----------------+-------------------------+-----------------------------------------------+ -| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | -| | format this yourself. | if no exception has occurred, ``None``. | -+----------------+-------------------------+-----------------------------------------------+ -| filename | ``%(filename)s`` | Filename portion of ``pathname``. | -+----------------+-------------------------+-----------------------------------------------+ -| funcName | ``%(funcName)s`` | Name of function containing the logging call. | -+----------------+-------------------------+-----------------------------------------------+ -| levelname | ``%(levelname)s`` | Text logging level for the message | -| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | -| | | ``'ERROR'``, ``'CRITICAL'``). | -+----------------+-------------------------+-----------------------------------------------+ -| levelno | ``%(levelno)s`` | Numeric logging level for the message | -| | | (:const:`DEBUG`, :const:`INFO`, | -| | | :const:`WARNING`, :const:`ERROR`, | -| | | :const:`CRITICAL`). | -+----------------+-------------------------+-----------------------------------------------+ -| lineno | ``%(lineno)d`` | Source line number where the logging call was | -| | | issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| message | ``%(message)s`` | The logged message, computed as ``msg % | -| | | args``. This is set when | -| | | :meth:`Formatter.format` is invoked. | -+----------------+-------------------------+-----------------------------------------------+ -| module | ``%(module)s`` | Module (name portion of ``filename``). | -+----------------+-------------------------+-----------------------------------------------+ -| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | -| | | :class:`LogRecord` was created. | -+----------------+-------------------------+-----------------------------------------------+ -| msg | You shouldn't need to | The format string passed in the original | -| | format this yourself. | logging call. Merged with ``args`` to | -| | | produce ``message``, or an arbitrary object | -| | | (see :ref:`arbitrary-object-messages`). | -+----------------+-------------------------+-----------------------------------------------+ -| name | ``%(name)s`` | Name of the logger used to log the call. | -+----------------+-------------------------+-----------------------------------------------+ -| pathname | ``%(pathname)s`` | Full pathname of the source file where the | -| | | logging call was issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| process | ``%(process)d`` | Process ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| processName | ``%(processName)s`` | Process name (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | -| | | created, relative to the time the logging | -| | | module was loaded. | -+----------------+-------------------------+-----------------------------------------------+ -| stack_info | You shouldn't need to | Stack frame information (where available) | -| | format this yourself. | from the bottom of the stack in the current | -| | | thread, up to and including the stack frame | -| | | of the logging call which resulted in the | -| | | creation of this record. | -+----------------+-------------------------+-----------------------------------------------+ -| thread | ``%(thread)d`` | Thread ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| threadName | ``%(threadName)s`` | Thread name (if available). | -+----------------+-------------------------+-----------------------------------------------+ - -.. versionchanged:: 3.1 - *processName* was added. - - -.. _logger-adapter: - -LoggerAdapter Objects ---------------------- - -:class:`LoggerAdapter` instances are used to conveniently pass contextual -information into logging calls. For a usage example, see the section on -:ref:`adding contextual information to your logging output `. - -.. class:: LoggerAdapter(logger, extra) - - Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. - - .. method:: process(msg, kwargs) - - Modifies the message and/or keyword arguments passed to a logging call in - order to insert contextual information. This implementation takes the object - passed as *extra* to the constructor and adds it to *kwargs* using key - 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the - (possibly modified) versions of the arguments passed in. - -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. - -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. - - -Thread Safety -------------- - -The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading -locks; there is one lock to serialize access to the module's shared data, and -each handler also creates a lock to serialize access to its underlying I/O. - -If you are implementing asynchronous signal handlers using the :mod:`signal` -module, you may not be able to use logging from within such handlers. This is -because lock implementations in the :mod:`threading` module are not always -re-entrant, and so cannot be invoked from such signal handlers. - - -Module-Level Functions ----------------------- - -In addition to the classes described above, there are a number of module-level -functions. - - -.. function:: getLogger(name=None) - - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. - - All calls to this function with a given name return the same logger instance. - This means that logger instances never need to be passed between different parts - of an application. - - -.. function:: getLoggerClass() - - Return either the standard :class:`Logger` class, or the last class passed to - :func:`setLoggerClass`. This function may be called from within a new class - definition, to ensure that installing a customized :class:`Logger` class will - not undo customizations already applied by other code. For example:: - - class MyLogger(logging.getLoggerClass()): - # ... override behaviour here - - -.. function:: getLogRecordFactory() - - Return a callable which is used to create a :class:`LogRecord`. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`setLogRecordFactory`, - to allow developers more control over how the :class:`LogRecord` - representing a logging event is constructed. - - See :func:`setLogRecordFactory` for more information about the how the - factory is called. - -.. function:: debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - -.. function:: info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. - - .. note:: There is an obsolete function ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - -.. function:: error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. - - -.. function:: exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging - message. This function should only be called from an exception handler. - -.. function:: log(level, msg, *args, **kwargs) - - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. - - .. note:: The above module-level convenience functions, which delegate to the - root logger, call :func:`basicConfig` to ensure that at least one handler - is available. Because of this, they should *not* be used in threads, - in versions of Python earlier than 2.7.1 and 3.2, unless at least one - handler has been added to the root logger *before* the threads are - started. In earlier versions of Python, due to a thread safety shortcoming - in :func:`basicConfig`, this can (under rare circumstances) lead to - handlers being added multiple times to the root logger, which can in turn - lead to multiple messages for the same event. - -.. function:: disable(level=CRITICAL) - - Provides an overriding level *level* for all loggers which takes precedence over - the logger's own level. When the need arises to temporarily throttle logging - output down across the whole application, this function can be useful. Its - effect is to disable all logging calls of severity *level* and below, so that - if you call it with a value of INFO, then all INFO and DEBUG events would be - discarded, whereas those of severity WARNING and above would be processed - according to the logger's effective level. If - ``logging.disable(logging.NOTSET)`` is called, it effectively removes this - overriding level, so that logging output again depends on the effective - levels of individual loggers. - - Note that if you have defined any custom logging level higher than - ``CRITICAL`` (this is not recommended), you won't be able to rely on the - default value for the *level* parameter, but will have to explicitly supply a - suitable value. - - .. versionchanged:: 3.7 - The *level* parameter was defaulted to level ``CRITICAL``. See - :issue:`28524` for more information about this change. - -.. function:: addLevelName(level, levelName) - - Associates level *level* with text *levelName* in an internal dictionary, which is - used to map numeric levels to a textual representation, for example when a - :class:`Formatter` formats a message. This function can also be used to define - your own levels. The only constraints are that all levels used must be - registered using this function, levels should be positive integers and they - should increase in increasing order of severity. - - .. note:: If you are thinking of defining your own levels, please see the - section on :ref:`custom-levels`. - -.. function:: getLevelName(level) - - Returns the textual representation of logging level *level*. If the level is one - of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, - :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you - have associated levels with names using :func:`addLevelName` then the name you - have associated with *level* is returned. If a numeric value corresponding to one - of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % level is returned. - - .. note:: Levels are internally integers (as they need to be compared in the - logging logic). This function is used to convert between an integer level - and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). - - .. versionchanged:: 3.4 - In Python versions earlier than 3.4, this function could also be passed a - text level, and would return the corresponding numeric value of the level. - This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. - -.. function:: makeLogRecord(attrdict) - - Creates and returns a new :class:`LogRecord` instance whose attributes are - defined by *attrdict*. This function is useful for taking a pickled - :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting - it as a :class:`LogRecord` instance at the receiving end. - - -.. function:: basicConfig(**kwargs) - - Does basic configuration for the logging system by creating a - :class:`StreamHandler` with a default :class:`Formatter` and adding it to the - root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, - :func:`error` and :func:`critical` will call :func:`basicConfig` automatically - if no handlers are defined for the root logger. - - This function does nothing if the root logger already has handlers - configured, unless the keyword argument *force* is set to ``True``. - - .. note:: This function should be called from the main thread - before other threads are started. In versions of Python prior to - 2.7.1 and 3.2, if this function is called from multiple threads, - it is possible (in rare circumstances) that a handler will be added - to the root logger more than once, leading to unexpected results - such as messages being duplicated in the log. - - The following keyword arguments are supported. - - .. tabularcolumns:: |l|L| - - +--------------+---------------------------------------------+ - | Format | Description | - +==============+=============================================+ - | *filename* | Specifies that a :class:`FileHandler` be | - | | created, using the specified filename, | - | | rather than a :class:`StreamHandler`. | - +--------------+---------------------------------------------+ - | *filemode* | If *filename* is specified, open the file | - | | in this :ref:`mode `. Defaults | - | | to ``'a'``. | - +--------------+---------------------------------------------+ - | *format* | Use the specified format string for the | - | | handler. Defaults to attributes | - | | ``levelname``, ``name`` and ``message`` | - | | separated by colons. | - +--------------+---------------------------------------------+ - | *datefmt* | Use the specified date/time format, as | - | | accepted by :func:`time.strftime`. | - +--------------+---------------------------------------------+ - | *style* | If *format* is specified, use this style | - | | for the format string. One of ``'%'``, | - | | ``'{'`` or ``'$'`` for :ref:`printf-style | - | | `, | - | | :meth:`str.format` or | - | | :class:`string.Template` respectively. | - | | Defaults to ``'%'``. | - +--------------+---------------------------------------------+ - | *level* | Set the root logger level to the specified | - | | :ref:`level `. | - +--------------+---------------------------------------------+ - | *stream* | Use the specified stream to initialize the | - | | :class:`StreamHandler`. Note that this | - | | argument is incompatible with *filename* - | - | | if both are present, a ``ValueError`` is | - | | raised. | - +--------------+---------------------------------------------+ - | *handlers* | If specified, this should be an iterable of | - | | already created handlers to add to the root | - | | logger. Any handlers which don't already | - | | have a formatter set will be assigned the | - | | default formatter created in this function. | - | | Note that this argument is incompatible | - | | with *filename* or *stream* - if both | - | | are present, a ``ValueError`` is raised. | - +--------------+---------------------------------------------+ - | *force* | If this keyword argument is specified as | - | | true, any existing handlers attached to the | - | | root logger are removed and closed, before | - | | carrying out the configuration as specified | - | | by the other arguments. | - +--------------+---------------------------------------------+ - | *encoding* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. | - +--------------+---------------------------------------------+ - | *errors* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. If not | - | | specified, the value 'backslashreplace' is | - | | used. Note that if ``None`` is specified, | - | | it will be passed as such to :func:`open`, | - | | which means that it will be treated the | - | | same as passing 'errors'. | - +--------------+---------------------------------------------+ - - .. versionchanged:: 3.2 - The *style* argument was added. - - .. versionchanged:: 3.3 - The *handlers* argument was added. Additional checks were added to - catch situations where incompatible arguments are specified (e.g. - *handlers* together with *stream* or *filename*, or *stream* - together with *filename*). - - .. versionchanged:: 3.8 - The *force* argument was added. - - .. versionchanged:: 3.9 - The *encoding* and *errors* arguments were added. - -.. function:: shutdown() - - Informs the logging system to perform an orderly shutdown by flushing and - closing all handlers. This should be called at application exit and no - further use of the logging system should be made after this call. - - When the logging module is imported, it registers this function as an exit - handler (see :mod:`atexit`), so normally there's no need to do that - manually. - - -.. function:: setLoggerClass(klass) - - Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This - function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. After this call, as at any other - time, do not instantiate loggers directly using the subclass: continue to use - the :func:`logging.getLogger` API to get your loggers. - - -.. function:: setLogRecordFactory(factory) - - Set a callable which is used to create a :class:`LogRecord`. - - :param factory: The factory callable to be used to instantiate a log record. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`getLogRecordFactory`, to - allow developers more control over how the :class:`LogRecord` representing - a logging event is constructed. - - The factory has the following signature: - - ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` - - :name: The logger name. - :level: The logging level (numeric). - :fn: The full pathname of the file where the logging call was made. - :lno: The line number in the file where the logging call was made. - :msg: The logging message. - :args: The arguments for the logging message. - :exc_info: An exception tuple, or ``None``. - :func: The name of the function or method which invoked the logging - call. - :sinfo: A stack traceback such as is provided by - :func:`traceback.print_stack`, showing the call hierarchy. - :kwargs: Additional keyword arguments. - - -Module-Level Attributes ------------------------ - -.. attribute:: lastResort - - A "handler of last resort" is available through this attribute. This - is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of - ``WARNING``, and is used to handle logging events in the absence of any - logging configuration. The end result is to just print the message to - ``sys.stderr``. This replaces the earlier error message saying that - "no handlers could be found for logger XYZ". If you need the earlier - behaviour for some reason, ``lastResort`` can be set to ``None``. - - .. versionadded:: 3.2 - -Integration with the warnings module ------------------------------------- - -The :func:`captureWarnings` function can be used to integrate :mod:`logging` -with the :mod:`warnings` module. - -.. function:: captureWarnings(capture) - - This function is used to turn the capture of warnings by logging on and - off. - - If *capture* is ``True``, warnings issued by the :mod:`warnings` module will - be redirected to the logging system. Specifically, a warning will be - formatted using :func:`warnings.formatwarning` and the resulting string - logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. - - If *capture* is ``False``, the redirection of warnings to the logging system - will stop, and warnings will be redirected to their original destinations - (i.e. those in effect before ``captureWarnings(True)`` was called). - - -.. seealso:: - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :pep:`282` - A Logging System - The proposal which described this feature for inclusion in the Python standard - library. - - `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the - package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard - library. diff --git a/Doc/library/readline.rst.bak b/Doc/library/readline.rst.bak deleted file mode 100644 index 3ff64885f7fced7..000000000000000 --- a/Doc/library/readline.rst.bak +++ /dev/null @@ -1,361 +0,0 @@ -:mod:`readline` --- GNU readline interface -========================================== - -.. module:: readline - :platform: Unix - :synopsis: GNU readline support for Python. - -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`readline` module defines a number of functions to facilitate -completion and reading/writing of history files from the Python interpreter. -This module can be used directly, or via the :mod:`rlcompleter` module, which -supports completion of Python identifiers at the interactive prompt. Settings -made using this module affect the behaviour of both the interpreter's -interactive prompt and the prompts offered by the built-in :func:`input` -function. - -Readline keybindings may be configured via an initialization file, typically -``.inputrc`` in your home directory. See `Readline Init File -`_ -in the GNU Readline manual for information about the format and -allowable constructs of that file, and the capabilities of the -Readline library in general. - -.. note:: - - The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used - at run time. - - The configuration file for ``libedit`` is different from that - of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. - - If you use *editline*/``libedit`` readline emulation on macOS, the - initialization file located in your home directory is named - ``.editrc``. For example, the following content in ``~/.editrc`` will - turn ON *vi* keybindings and TAB completion:: - - python:bind -v - python:bind ^I rl_complete - - -Init file ---------- - -The following functions relate to the init file and user configuration: - - -.. function:: parse_and_bind(string) - - Execute the init line provided in the *string* argument. This calls - :c:func:`rl_parse_and_bind` in the underlying library. - - -.. function:: read_init_file([filename]) - - Execute a readline initialization file. The default filename is the last filename - used. This calls :c:func:`rl_read_init_file` in the underlying library. - - -Line buffer ------------ - -The following functions operate on the line buffer: - - -.. function:: get_line_buffer() - - Return the current contents of the line buffer (:c:data:`rl_line_buffer` - in the underlying library). - - -.. function:: insert_text(string) - - Insert text into the line buffer at the cursor position. This calls - :c:func:`rl_insert_text` in the underlying library, but ignores - the return value. - - -.. function:: redisplay() - - Change what's displayed on the screen to reflect the current contents of the - line buffer. This calls :c:func:`rl_redisplay` in the underlying library. - - -History file ------------- - -The following functions operate on a history file: - - -.. function:: read_history_file([filename]) - - Load a readline history file, and append it to the history list. - The default filename is :file:`~/.history`. This calls - :c:func:`read_history` in the underlying library. - - -.. function:: write_history_file([filename]) - - Save the history list to a readline history file, overwriting any - existing file. The default filename is :file:`~/.history`. This calls - :c:func:`write_history` in the underlying library. - - -.. function:: append_history_file(nelements[, filename]) - - Append the last *nelements* items of history to a file. The default filename is - :file:`~/.history`. The file must already exist. This calls - :c:func:`append_history` in the underlying library. This function - only exists if Python was compiled for a version of the library - that supports it. - - .. versionadded:: 3.5 - - -.. function:: get_history_length() - set_history_length(length) - - Set or return the desired number of lines to save in the history file. - The :func:`write_history_file` function uses this value to truncate - the history file, by calling :c:func:`history_truncate_file` in - the underlying library. Negative values imply - unlimited history file size. - - -History list ------------- - -The following functions operate on a global history list: - - -.. function:: clear_history() - - Clear the current history. This calls :c:func:`clear_history` in the - underlying library. The Python function only exists if Python was - compiled for a version of the library that supports it. - - -.. function:: get_current_history_length() - - Return the number of items currently in the history. (This is different from - :func:`get_history_length`, which returns the maximum number of lines that will - be written to a history file.) - - -.. function:: get_history_item(index) - - Return the current contents of history item at *index*. The item index - is one-based. This calls :c:func:`history_get` in the underlying library. - - -.. function:: remove_history_item(pos) - - Remove history item specified by its position from the history. - The position is zero-based. This calls :c:func:`remove_history` in - the underlying library. - - -.. function:: replace_history_item(pos, line) - - Replace history item specified by its position with *line*. - The position is zero-based. This calls :c:func:`replace_history_entry` - in the underlying library. - - -.. function:: add_history(line) - - Append *line* to the history buffer, as if it was the last line typed. - This calls :c:func:`add_history` in the underlying library. - - -.. function:: set_auto_history(enabled) - - Enable or disable automatic calls to :c:func:`add_history` when reading - input via readline. The *enabled* argument should be a Boolean value - that when true, enables auto history, and that when false, disables - auto history. - - .. versionadded:: 3.6 - - .. impl-detail:: - Auto history is enabled by default, and changes to this do not persist - across multiple sessions. - - -Startup hooks -------------- - - -.. function:: set_startup_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_startup_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any function - already installed is removed. The hook is called with no - arguments just before readline prints the first prompt. - - -.. function:: set_pre_input_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_pre_input_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any - function already installed is removed. The hook is called - with no arguments after the first prompt has been printed and just before - readline starts reading input characters. This function only exists - if Python was compiled for a version of the library that supports it. - - -Completion ----------- - -The following functions relate to implementing a custom word completion -function. This is typically operated by the Tab key, and can suggest and -automatically complete a word being typed. By default, Readline is set up -to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used -with a custom completer, a different set of word delimiters should be set. - - -.. function:: set_completer([function]) - - Set or remove the completer function. If *function* is specified, it will be - used as the new completer function; if omitted or ``None``, any completer - function already installed is removed. The completer function is called as - ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it - returns a non-string value. It should return the next possible completion - starting with *text*. - - The installed completer function is invoked by the *entry_func* callback - passed to :c:func:`rl_completion_matches` in the underlying library. - The *text* string comes from the first parameter to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. - - -.. function:: get_completer() - - Get the completer function, or ``None`` if no completer function has been set. - - -.. function:: get_completion_type() - - Get the type of completion being attempted. This returns the - :c:data:`rl_completion_type` variable in the underlying library as - an integer. - - -.. function:: get_begidx() - get_endidx() - - Get the beginning or ending index of the completion scope. - These indexes are the *start* and *end* arguments passed to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. The values may be different in the same - input editing scenario based on the underlying C readline implemtation. - Ex: libedit is known to behave differently than libreadline. - - -.. function:: set_completer_delims(string) - get_completer_delims() - - Set or get the word delimiters for completion. These determine the - start of the word to be considered for completion (the completion scope). - These functions access the :c:data:`rl_completer_word_break_characters` - variable in the underlying library. - - -.. function:: set_completion_display_matches_hook([function]) - - Set or remove the completion display function. If *function* is - specified, it will be used as the new completion display function; - if omitted or ``None``, any completion display function already - installed is removed. This sets or clears the - :c:data:`rl_completion_display_matches_hook` callback in the - underlying library. The completion display function is called as - ``function(substitution, [matches], longest_match_length)`` once - each time matches need to be displayed. - - -.. _readline-example: - -Example -------- - -The following example demonstrates how to use the :mod:`readline` module's -history reading and writing functions to automatically load and save a history -file named :file:`.python_history` from the user's home directory. The code -below would normally be executed automatically during interactive sessions -from the user's :envvar:`PYTHONSTARTUP` file. :: - - import atexit - import os - import readline - - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - try: - readline.read_history_file(histfile) - # default history len is -1 (infinite), which may grow unruly - readline.set_history_length(1000) - except FileNotFoundError: - pass - - atexit.register(readline.write_history_file, histfile) - -This code is actually automatically run when Python is run in -:ref:`interactive mode ` (see :ref:`rlcompleter-config`). - -The following example achieves the same goal but supports concurrent interactive -sessions, by only appending the new history. :: - - import atexit - import os - import readline - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - - try: - readline.read_history_file(histfile) - h_len = readline.get_current_history_length() - except FileNotFoundError: - open(histfile, 'wb').close() - h_len = 0 - - def save(prev_h_len, histfile): - new_h_len = readline.get_current_history_length() - readline.set_history_length(1000) - readline.append_history_file(new_h_len - prev_h_len, histfile) - atexit.register(save, h_len, histfile) - -The following example extends the :class:`code.InteractiveConsole` class to -support history save/restore. :: - - import atexit - import code - import os - import readline - - class HistoryConsole(code.InteractiveConsole): - def __init__(self, locals=None, filename="", - histfile=os.path.expanduser("~/.console-history")): - code.InteractiveConsole.__init__(self, locals, filename) - self.init_history(histfile) - - def init_history(self, histfile): - readline.parse_and_bind("tab: complete") - if hasattr(readline, "read_history_file"): - try: - readline.read_history_file(histfile) - except FileNotFoundError: - pass - atexit.register(self.save_history, histfile) - - def save_history(self, histfile): - readline.set_history_length(1000) - readline.write_history_file(histfile) diff --git a/Doc/library/sqlite3.rst.bak b/Doc/library/sqlite3.rst.bak deleted file mode 100644 index 172ce6c6bb03ba3..000000000000000 --- a/Doc/library/sqlite3.rst.bak +++ /dev/null @@ -1,1094 +0,0 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ - -.. module:: sqlite3 - :synopsis: A DB-API 2.0 implementation using SQLite 3.x. - -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/sqlite3/` - --------------- - -SQLite is a C library that provides a lightweight disk-based database that -doesn't require a separate server process and allows accessing the database -using a nonstandard variant of the SQL query language. Some applications can use -SQLite for internal data storage. It's also possible to prototype an -application using SQLite and then port the code to a larger database such as -PostgreSQL or Oracle. - -The sqlite3 module was written by Gerhard Häring. It provides a SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.7.15 or newer. - -To use the module, you must first create a :class:`Connection` object that -represents the database. Here the data will be stored in the -:file:`example.db` file:: - - import sqlite3 - con = sqlite3.connect('example.db') - -You can also supply the special name ``:memory:`` to create a database in RAM. - -Once you have a :class:`Connection`, you can create a :class:`Cursor` object -and call its :meth:`~Cursor.execute` method to perform SQL commands:: - - cur = con.cursor() - - # Create table - cur.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') - - # Insert a row of data - cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") - - # Save (commit) the changes - con.commit() - - # We can also close the connection if we are done with it. - # Just be sure any changes have been committed or they will be lost. - con.close() - -The data you've saved is persistent and is available in subsequent sessions:: - - import sqlite3 - con = sqlite3.connect('example.db') - cur = con.cursor() - -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see https://xkcd.com/327/ for humorous example of what can go wrong). - -Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder -wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`~Cursor.execute` method. (Other database -modules may use a different placeholder, such as ``%s`` or ``:1``.) For -example:: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) - - # Do this instead - t = ('RHAT',) - cur.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(cur.fetchone()) - - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) - -To retrieve data after executing a SELECT statement, you can either treat the -cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to -retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the -matching rows. - -This example uses the iterator form:: - - >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): - print(row) - - ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) - ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) - ('2006-04-06', 'SELL', 'IBM', 500, 53.0) - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) - - -.. seealso:: - - https://www.sqlite.org - The SQLite web page; the documentation describes the syntax and the - available data types for the supported SQL dialect. - - https://www.w3schools.com/sql/ - Tutorial, reference and examples for learning SQL syntax. - - :pep:`249` - Database API Specification 2.0 - PEP written by Marc-André Lemburg. - - -.. _sqlite3-module-contents: - -Module functions and constants ------------------------------- - - -.. data:: version - - The version number of this module, as a string. This is not the version of - the SQLite library. - - -.. data:: version_info - - The version number of this module, as a tuple of integers. This is not the - version of the SQLite library. - - -.. data:: sqlite_version - - The version number of the run-time SQLite library, as a string. - - -.. data:: sqlite_version_info - - The version number of the run-time SQLite library, as a tuple of integers. - - -.. data:: PARSE_DECLTYPES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting it makes the :mod:`sqlite3` module parse the declared type for each - column it returns. It will parse out the first word of the declared type, - i. e. for "integer primary key", it will parse out "integer", or for - "number(10)" it will parse out "number". Then for that column, it will look - into the converters dictionary and use the converter function registered for - that type there. - - -.. data:: PARSE_COLNAMES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting this makes the SQLite interface parse the column name for each column it - returns. It will look for a string formed [mytype] in there, and then decide - that 'mytype' is the type of the column. It will try to find an entry of - 'mytype' in the converters dictionary and then use the converter function found - there to return the value. The column name found in :attr:`Cursor.description` - does not include the type, i. e. if you use something like - ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out - everything until the first ``'['`` for the column name and strip - the preceeding space: the column name would simply be "Expiration date". - - -.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) - - Opens a connection to the SQLite database file *database*. By default returns a - :class:`Connection` object, unless a custom *factory* is given. - - *database* is a :term:`path-like object` giving the pathname (absolute or - relative to the current working directory) of the database file to be opened. - You can use ``":memory:"`` to open a database connection to a database that - resides in RAM instead of on disk. - - When a database is accessed by multiple connections, and one of the processes - modifies the database, the SQLite database is locked until that transaction is - committed. The *timeout* parameter specifies how long the connection should wait - for the lock to go away until raising an exception. The default for the timeout - parameter is 5.0 (five seconds). - - For the *isolation_level* parameter, please see the - :attr:`~Connection.isolation_level` property of :class:`Connection` objects. - - SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If - you want to use other types you must add support for them yourself. The - *detect_types* parameter and the using custom **converters** registered with the - module-level :func:`register_converter` function allow you to easily do that. - - *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to - any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn - type detection on. Due to SQLite behaviour, types can't be detected for generated - fields (for example ``max(data)``), even when *detect_types* parameter is set. In - such case, the returned type is :class:`str`. - - By default, *check_same_thread* is :const:`True` and only the creating thread may - use the connection. If set :const:`False`, the returned connection may be shared - across multiple threads. When using multiple threads with the same connection - writing operations should be serialized by the user to avoid data corruption. - - By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the - connect call. You can, however, subclass the :class:`Connection` class and make - :func:`connect` use your class instead by providing your class for the *factory* - parameter. - - Consult the section :ref:`sqlite3-types` of this manual for details. - - The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing - overhead. If you want to explicitly set the number of statements that are cached - for the connection, you can set the *cached_statements* parameter. The currently - implemented default is to cache 100 statements. - - If *uri* is true, *database* is interpreted as a URI. This allows you - to specify options. For example, to open a database in read-only mode - you can use:: - - db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) - - More information about this feature, including a list of recognized options, can - be found in the `SQLite URI documentation `_. - - .. audit-event:: sqlite3.connect database sqlite3.connect - - .. versionchanged:: 3.4 - Added the *uri* parameter. - - .. versionchanged:: 3.7 - *database* can now also be a :term:`path-like object`, not only a string. - - -.. function:: register_converter(typename, callable) - - Registers a callable to convert a bytestring from the database into a custom - Python type. The callable will be invoked for all database values that are of - the type *typename*. Confer the parameter *detect_types* of the :func:`connect` - function for how the type detection works. Note that *typename* and the name of - the type in your query are matched in case-insensitive manner. - - -.. function:: register_adapter(type, callable) - - Registers a callable to convert the custom Python type *type* into one of - SQLite's supported types. The callable *callable* accepts as single parameter - the Python value, and must return a value of the following types: int, - float, str or bytes. - - -.. function:: complete_statement(sql) - - Returns :const:`True` if the string *sql* contains one or more complete SQL - statements terminated by semicolons. It does not verify that the SQL is - syntactically correct, only that there are no unclosed string literals and the - statement is terminated by a semicolon. - - This can be used to build a shell for SQLite, as in the following example: - - - .. literalinclude:: ../includes/sqlite3/complete_statement.py - - -.. function:: enable_callback_tracebacks(flag) - - By default you will not get any tracebacks in user-defined functions, - aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you will - get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to - disable the feature again. - - -.. _sqlite3-connection-objects: - -Connection Objects ------------------- - -.. class:: Connection - - A SQLite database connection has the following attributes and methods: - - .. attribute:: isolation_level - - Get or set the current default isolation level. :const:`None` for autocommit mode or - one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section - :ref:`sqlite3-controlling-transactions` for a more detailed explanation. - - .. attribute:: in_transaction - - :const:`True` if a transaction is active (there are uncommitted changes), - :const:`False` otherwise. Read-only attribute. - - .. versionadded:: 3.2 - - .. method:: cursor(factory=Cursor) - - The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a callable returning an instance of :class:`Cursor` - or its subclasses. - - .. method:: commit() - - This method commits the current transaction. If you don't call this method, - anything you did since the last call to ``commit()`` is not visible from - other database connections. If you wonder why you don't see the data you've - written to the database, please check you didn't forget to call this method. - - .. method:: rollback() - - This method rolls back any changes to the database since the last call to - :meth:`commit`. - - .. method:: close() - - This closes the database connection. Note that this does not automatically - call :meth:`commit`. If you just close your database connection without - calling :meth:`commit` first, your changes will be lost! - - .. method:: execute(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by calling - the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.execute` method with the *parameters* given, and returns - the cursor. - - .. method:: executemany(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executemany` method with the *parameters* given, and - returns the cursor. - - .. method:: executescript(sql_script) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executescript` method with the given *sql_script*, and - returns the cursor. - - .. method:: create_function(name, num_params, func, *, deterministic=False) - - Creates a user-defined function that you can later use from within SQL - statements under the function name *name*. *num_params* is the number of - parameters the function accepts (if *num_params* is -1, the function may - take any number of arguments), and *func* is a Python callable that is - called as the SQL function. If *deterministic* is true, the created function - is marked as `deterministic `_, which - allows SQLite to perform additional optimizations. This flag is supported by - SQLite 3.8.3 or higher, :exc:`NotSupportedError` will be raised if used - with older versions. - - The function can return any of the types supported by SQLite: bytes, str, int, - float and ``None``. - - .. versionchanged:: 3.8 - The *deterministic* parameter was added. - - Example: - - .. literalinclude:: ../includes/sqlite3/md5func.py - - - .. method:: create_aggregate(name, num_params, aggregate_class) - - Creates a user-defined aggregate function. - - The aggregate class must implement a ``step`` method, which accepts the number - of parameters *num_params* (if *num_params* is -1, the function may take - any number of arguments), and a ``finalize`` method which will return the - final result of the aggregate. - - The ``finalize`` method can return any of the types supported by SQLite: - bytes, str, int, float and ``None``. - - Example: - - .. literalinclude:: ../includes/sqlite3/mysumaggr.py - - - .. method:: create_collation(name, callable) - - Creates a collation with the specified *name* and *callable*. The callable will - be passed two string arguments. It should return -1 if the first is ordered - lower than the second, 0 if they are ordered equal and 1 if the first is ordered - higher than the second. Note that this controls sorting (ORDER BY in SQL) so - your comparisons don't affect other SQL operations. - - Note that the callable will get its parameters as Python bytestrings, which will - normally be encoded in UTF-8. - - The following example shows a custom collation that sorts "the wrong way": - - .. literalinclude:: ../includes/sqlite3/collation_reverse.py - - To remove a collation, call ``create_collation`` with ``None`` as callable:: - - con.create_collation("reverse", None) - - - .. method:: interrupt() - - You can call this method from a different thread to abort any queries that might - be executing on the connection. The query will then abort and the caller will - get an exception. - - - .. method:: set_authorizer(authorizer_callback) - - This routine registers a callback. The callback is invoked for each attempt to - access a column of a table in the database. The callback should return - :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL - statement should be aborted with an error and :const:`SQLITE_IGNORE` if the - column should be treated as a NULL value. These constants are available in the - :mod:`sqlite3` module. - - The first argument to the callback signifies what kind of operation is to be - authorized. The second and third argument will be arguments or :const:`None` - depending on the first argument. The 4th argument is the name of the database - ("main", "temp", etc.) if applicable. The 5th argument is the name of the - inner-most trigger or view that is responsible for the access attempt or - :const:`None` if this access attempt is directly from input SQL code. - - Please consult the SQLite documentation about the possible values for the first - argument and the meaning of the second and third argument depending on the first - one. All necessary constants are available in the :mod:`sqlite3` module. - - - .. method:: set_progress_handler(handler, n) - - This routine registers a callback. The callback is invoked for every *n* - instructions of the SQLite virtual machine. This is useful if you want to - get called from SQLite during long-running operations, for example to update - a GUI. - - If you want to clear any previously installed progress handler, call the - method with :const:`None` for *handler*. - - Returning a non-zero value from the handler function will terminate the - currently executing query and cause it to raise an :exc:`OperationalError` - exception. - - - .. method:: set_trace_callback(trace_callback) - - Registers *trace_callback* to be called for each SQL statement that is - actually executed by the SQLite backend. - - The only argument passed to the callback is the statement (as string) that - is being executed. The return value of the callback is ignored. Note that - the backend does not only run statements passed to the :meth:`Cursor.execute` - methods. Other sources include the transaction management of the Python - module and the execution of triggers defined in the current database. - - Passing :const:`None` as *trace_callback* will disable the trace callback. - - .. versionadded:: 3.3 - - - .. method:: enable_load_extension(enabled) - - This routine allows/disallows the SQLite engine to load SQLite extensions - from shared libraries. SQLite extensions can define new functions, - aggregates or whole new virtual table implementations. One well-known - extension is the fulltext-search extension distributed with SQLite. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. literalinclude:: ../includes/sqlite3/load_extension.py - - .. method:: load_extension(path) - - This routine loads a SQLite extension from a shared library. You have to - enable extension loading with :meth:`enable_load_extension` before you can - use this routine. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. attribute:: row_factory - - You can change this attribute to a callable that accepts the cursor and the - original row as a tuple and will return the real result row. This way, you can - implement more advanced ways of returning results, such as returning an object - that can also access columns by name. - - Example: - - .. literalinclude:: ../includes/sqlite3/row_factory.py - - If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. - - .. XXX what's a db_row-based solution? - - - .. attribute:: text_factory - - Using this attribute you can control what objects are returned for the ``TEXT`` - data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. - - You can also set it to any other callable that accepts a single bytestring - parameter and returns the resulting object. - - See the following example code for illustration: - - .. literalinclude:: ../includes/sqlite3/text_factory.py - - - .. attribute:: total_changes - - Returns the total number of database rows that have been modified, inserted, or - deleted since the database connection was opened. - - - .. method:: iterdump - - Returns an iterator to dump the database in an SQL text format. Useful when - saving an in-memory database for later restoration. This function provides - the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` - shell. - - Example:: - - # Convert file existing_db.db to SQL dump file dump.sql - import sqlite3 - - con = sqlite3.connect('existing_db.db') - with open('dump.sql', 'w') as f: - for line in con.iterdump(): - f.write('%s\n' % line) - con.close() - - - .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) - - This method makes a backup of a SQLite database even while it's being accessed - by other clients, or concurrently by the same connection. The copy will be - written into the mandatory argument *target*, that must be another - :class:`Connection` instance. - - By default, or when *pages* is either ``0`` or a negative integer, the entire - database is copied in a single step; otherwise the method performs a loop - copying up to *pages* pages at a time. - - If *progress* is specified, it must either be ``None`` or a callable object that - will be executed at each iteration with three integer arguments, respectively - the *status* of the last iteration, the *remaining* number of pages still to be - copied and the *total* number of pages. - - The *name* argument specifies the database name that will be copied: it must be - a string containing either ``"main"``, the default, to indicate the main - database, ``"temp"`` to indicate the temporary database or the name specified - after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached - database. - - The *sleep* argument specifies the number of seconds to sleep by between - successive attempts to backup remaining pages, can be specified either as an - integer or a floating point value. - - Example 1, copy an existing database into another:: - - import sqlite3 - - def progress(status, remaining, total): - print(f'Copied {total-remaining} of {total} pages...') - - con = sqlite3.connect('existing_db.db') - bck = sqlite3.connect('backup.db') - with bck: - con.backup(bck, pages=1, progress=progress) - bck.close() - con.close() - - Example 2, copy an existing database into a transient copy:: - - import sqlite3 - - source = sqlite3.connect('existing_db.db') - dest = sqlite3.connect(':memory:') - source.backup(dest) - - .. versionadded:: 3.7 - - -.. _sqlite3-cursor-objects: - -Cursor Objects --------------- - -.. class:: Cursor - - A :class:`Cursor` instance has the following attributes and methods. - - .. index:: single: ? (question mark); in SQL statements - .. index:: single: : (colon); in SQL statements - - .. method:: execute(sql[, parameters]) - - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). - - Here's an example of both styles: - - .. literalinclude:: ../includes/sqlite3/execute_1.py - - :meth:`execute` will only execute a single SQL statement. If you try to execute - more than one statement with it, it will raise a :exc:`.Warning`. Use - :meth:`executescript` if you want to execute multiple SQL statements with one - call. - - - .. method:: executemany(sql, seq_of_parameters) - - Executes an SQL command against all parameter sequences or mappings found in - the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows - using an :term:`iterator` yielding parameters instead of a sequence. - - .. literalinclude:: ../includes/sqlite3/executemany_1.py - - Here's a shorter example using a :term:`generator`: - - .. literalinclude:: ../includes/sqlite3/executemany_2.py - - - .. method:: executescript(sql_script) - - This is a nonstandard convenience method for executing multiple SQL statements - at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. - - *sql_script* can be an instance of :class:`str`. - - Example: - - .. literalinclude:: ../includes/sqlite3/executescript.py - - - .. method:: fetchone() - - Fetches the next row of a query result set, returning a single sequence, - or :const:`None` when no more data is available. - - - .. method:: fetchmany(size=cursor.arraysize) - - Fetches the next set of rows of a query result, returning a list. An empty - list is returned when no more rows are available. - - The number of rows to fetch per call is specified by the *size* parameter. - If it is not given, the cursor's arraysize determines the number of rows - to be fetched. The method should try to fetch as many rows as indicated by - the size parameter. If this is not possible due to the specified number of - rows not being available, fewer rows may be returned. - - Note there are performance considerations involved with the *size* parameter. - For optimal performance, it is usually best to use the arraysize attribute. - If the *size* parameter is used, then it is best for it to retain the same - value from one :meth:`fetchmany` call to the next. - - .. method:: fetchall() - - Fetches all (remaining) rows of a query result, returning a list. Note that - the cursor's arraysize attribute can affect the performance of this operation. - An empty list is returned when no rows are available. - - .. method:: close() - - Close the cursor now (rather than whenever ``__del__`` is called). - - The cursor will be unusable from this point forward; a :exc:`ProgrammingError` - exception will be raised if any operation is attempted with the cursor. - - .. attribute:: rowcount - - Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this - attribute, the database engine's own support for the determination of "rows - affected"/"rows selected" is quirky. - - For :meth:`executemany` statements, the number of modifications are summed up - into :attr:`rowcount`. - - As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in - case no ``executeXX()`` has been performed on the cursor or the rowcount of the - last operation is not determinable by the interface". This includes ``SELECT`` - statements because we cannot determine the number of rows a query produced - until all rows were fetched. - - .. attribute:: lastrowid - - This read-only attribute provides the rowid of the last modified row. It is - only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the - :meth:`execute` method. For operations other than ``INSERT`` or - ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is - set to :const:`None`. - - If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous - successful rowid is returned. - - .. versionchanged:: 3.6 - Added support for the ``REPLACE`` statement. - - .. attribute:: arraysize - - Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. - The default value is 1 which means a single row would be fetched per call. - - .. attribute:: description - - This read-only attribute provides the column names of the last query. To - remain compatible with the Python DB API, it returns a 7-tuple for each - column where the last six items of each tuple are :const:`None`. - - It is set for ``SELECT`` statements without any matching rows as well. - - .. attribute:: connection - - This read-only attribute provides the SQLite database :class:`Connection` - used by the :class:`Cursor` object. A :class:`Cursor` object created by - calling :meth:`con.cursor() ` will have a - :attr:`connection` attribute that refers to *con*:: - - >>> con = sqlite3.connect(":memory:") - >>> cur = con.cursor() - >>> cur.connection == con - True - -.. _sqlite3-row-objects: - -Row Objects ------------ - -.. class:: Row - - A :class:`Row` instance serves as a highly optimized - :attr:`~Connection.row_factory` for :class:`Connection` objects. - It tries to mimic a tuple in most of its features. - - It supports mapping access by column name and index, iteration, - representation, equality testing and :func:`len`. - - If two :class:`Row` objects have exactly the same columns and their - members are equal, they compare equal. - - .. method:: keys - - This method returns a list of column names. Immediately after a query, - it is the first member of each tuple in :attr:`Cursor.description`. - - .. versionchanged:: 3.5 - Added support of slicing. - -Let's assume we initialize a table as in the example given above:: - - con = sqlite3.connect(":memory:") - cur = con.cursor() - cur.execute('''create table stocks - (date text, trans text, symbol text, - qty real, price real)''') - cur.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") - con.commit() - cur.close() - -Now we plug :class:`Row` in:: - - >>> con.row_factory = sqlite3.Row - >>> cur = con.cursor() - >>> cur.execute('select * from stocks') - - >>> r = cur.fetchone() - >>> type(r) - - >>> tuple(r) - ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) - >>> len(r) - 5 - >>> r[2] - 'RHAT' - >>> r.keys() - ['date', 'trans', 'symbol', 'qty', 'price'] - >>> r['qty'] - 100.0 - >>> for member in r: - ... print(member) - ... - 2006-01-05 - BUY - RHAT - 100.0 - 35.14 - - -.. _sqlite3-exceptions: - -Exceptions ----------- - -.. exception:: Warning - - A subclass of :exc:`Exception`. - -.. exception:: Error - - The base class of the other exceptions in this module. It is a subclass - of :exc:`Exception`. - -.. exception:: DatabaseError - - Exception raised for errors that are related to the database. - -.. exception:: IntegrityError - - Exception raised when the relational integrity of the database is affected, - e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. - -.. exception:: ProgrammingError - - Exception raised for programming errors, e.g. table not found or already - exists, syntax error in the SQL statement, wrong number of parameters - specified, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: OperationalError - - Exception raised for errors that are related to the database's operation - and not necessarily under the control of the programmer, e.g. an unexpected - disconnect occurs, the data source name is not found, a transaction could - not be processed, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: NotSupportedError - - Exception raised in case a method or database API was used which is not - supported by the database, e.g. calling the :meth:`~Connection.rollback` - method on a connection that does not support transaction or has - transactions turned off. It is a subclass of :exc:`DatabaseError`. - - -.. _sqlite3-types: - -SQLite and Python types ------------------------ - - -Introduction -^^^^^^^^^^^^ - -SQLite natively supports the following types: ``NULL``, ``INTEGER``, -``REAL``, ``TEXT``, ``BLOB``. - -The following Python types can thus be sent to SQLite without any problem: - -+-------------------------------+-------------+ -| Python type | SQLite type | -+===============================+=============+ -| :const:`None` | ``NULL`` | -+-------------------------------+-------------+ -| :class:`int` | ``INTEGER`` | -+-------------------------------+-------------+ -| :class:`float` | ``REAL`` | -+-------------------------------+-------------+ -| :class:`str` | ``TEXT`` | -+-------------------------------+-------------+ -| :class:`bytes` | ``BLOB`` | -+-------------------------------+-------------+ - - -This is how SQLite types are converted to Python types by default: - -+-------------+----------------------------------------------+ -| SQLite type | Python type | -+=============+==============================================+ -| ``NULL`` | :const:`None` | -+-------------+----------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+----------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+----------------------------------------------+ -| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | -| | :class:`str` by default | -+-------------+----------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+----------------------------------------------+ - -The type system of the :mod:`sqlite3` module is extensible in two ways: you can -store additional Python types in a SQLite database via object adaptation, and -you can let the :mod:`sqlite3` module convert SQLite types to different Python -types via converters. - - -Using adapters to store additional Python types in SQLite databases -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As described before, SQLite supports only a limited set of types natively. To -use other Python types with SQLite, you must **adapt** them to one of the -sqlite3 module's supported types for SQLite: one of NoneType, int, float, -str, bytes. - -There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python -type to one of the supported ones. - - -Letting your object adapt itself -"""""""""""""""""""""""""""""""" - -This is a good approach if you write the class yourself. Let's suppose you have -a class like this:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - -Now you want to store the point in a single SQLite column. First you'll have to -choose one of the supported types to be used for representing the point. -Let's just use str and separate the coordinates using a semicolon. Then you need -to give your class a method ``__conform__(self, protocol)`` which must return -the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_1.py - - -Registering an adapter callable -""""""""""""""""""""""""""""""" - -The other possibility is to create a function that converts the type to the -string representation and register the function with :meth:`register_adapter`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_2.py - -The :mod:`sqlite3` module has two default adapters for Python's built-in -:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose -we want to store :class:`datetime.datetime` objects not in ISO representation, -but as a Unix timestamp. - -.. literalinclude:: ../includes/sqlite3/adapter_datetime.py - - -Converting SQLite values to custom Python types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Writing an adapter lets you send custom Python types to SQLite. But to make it -really useful we need to make the Python to SQLite to Python roundtrip work. - -Enter converters. - -Let's go back to the :class:`Point` class. We stored the x and y coordinates -separated via semicolons as strings in SQLite. - -First, we'll define a converter function that accepts the string as a parameter -and constructs a :class:`Point` object from it. - -.. note:: - - Converter functions **always** get called with a :class:`bytes` object, no - matter under which data type you sent the value to SQLite. - -:: - - def convert_point(s): - x, y = map(float, s.split(b";")) - return Point(x, y) - -Now you need to make the :mod:`sqlite3` module know that what you select from -the database is actually a point. There are two ways of doing this: - -* Implicitly via the declared type - -* Explicitly via the column name - -Both ways are described in section :ref:`sqlite3-module-contents`, in the entries -for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. - -The following example illustrates both approaches. - -.. literalinclude:: ../includes/sqlite3/converter_point.py - - -Default adapters and converters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are default adapters for the date and datetime types in the datetime -module. They will be sent as ISO dates/ISO timestamps to SQLite. - -The default converters are registered under the name "date" for -:class:`datetime.date` and under the name "timestamp" for -:class:`datetime.datetime`. - -This way, you can use date/timestamps from Python without any additional -fiddling in most cases. The format of the adapters is also compatible with the -experimental SQLite date/time functions. - -The following example demonstrates this. - -.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py - -If a timestamp stored in SQLite has a fractional part longer than 6 -numbers, its value will be truncated to microsecond precision by the -timestamp converter. - - -.. _sqlite3-controlling-transactions: - -Controlling Transactions ------------------------- - -The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, -but the Python :mod:`sqlite3` module by default does not. - -``autocommit`` mode means that statements that modify the database take effect -immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` -mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the -outermost transaction, turns ``autocommit`` mode back on. - -The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement -implicitly before a Data Modification Language (DML) statement (i.e. -``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). - -You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly -executes via the *isolation_level* parameter to the :func:`connect` -call, or via the :attr:`isolation_level` property of connections. -If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is -equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` -and ``EXCLUSIVE``. - -You can disable the :mod:`sqlite3` module's implicit transaction management by -setting :attr:`isolation_level` to ``None``. This will leave the underlying -``sqlite3`` library operating in ``autocommit`` mode. You can then completely -control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, -``SAVEPOINT``, and ``RELEASE`` statements in your code. - -.. versionchanged:: 3.6 - :mod:`sqlite3` used to implicitly commit an open transaction before DDL - statements. This is no longer the case. - - -Using :mod:`sqlite3` efficiently --------------------------------- - - -Using shortcut methods -^^^^^^^^^^^^^^^^^^^^^^ - -Using the nonstandard :meth:`execute`, :meth:`executemany` and -:meth:`executescript` methods of the :class:`Connection` object, your code can -be written more concisely because you don't have to create the (often -superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` -objects are created implicitly and these shortcut methods return the cursor -objects. This way, you can execute a ``SELECT`` statement and iterate over it -directly using only a single call on the :class:`Connection` object. - -.. literalinclude:: ../includes/sqlite3/shortcut_methods.py - - -Accessing columns by name instead of by index -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -One useful feature of the :mod:`sqlite3` module is the built-in -:class:`sqlite3.Row` class designed to be used as a row factory. - -Rows wrapped with this class can be accessed both by index (like tuples) and -case-insensitively by name: - -.. literalinclude:: ../includes/sqlite3/rowclass.py - - -Using the connection as a context manager -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Connection objects can be used as context managers -that automatically commit or rollback transactions. In the event of an -exception, the transaction is rolled back; otherwise, the transaction is -committed: - -.. literalinclude:: ../includes/sqlite3/ctx_manager.py - - -.. rubric:: Footnotes - -.. [#f1] The sqlite3 module is not built with loadable extension support by - default, because some platforms (notably Mac OS X) have SQLite - libraries which are compiled without this feature. To get loadable - extension support, you must pass --enable-loadable-sqlite-extensions to - configure. diff --git a/Doc/library/statistics.rst.bak b/Doc/library/statistics.rst.bak deleted file mode 100644 index 6b6d3154a288100..000000000000000 --- a/Doc/library/statistics.rst.bak +++ /dev/null @@ -1,879 +0,0 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= - -.. module:: statistics - :synopsis: Mathematical statistics functions - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/statistics.py` - -.. testsetup:: * - - from statistics import * - __name__ = '' - --------------- - -This module provides functions for calculating mathematical statistics of -numeric (:class:`~numbers.Real`-valued) data. - -The module is not intended to be a competitor to third-party libraries such -as `NumPy `_, `SciPy `_, or -proprietary full-featured statistics packages aimed at professional -statisticians such as Minitab, SAS and Matlab. It is aimed at the level of -graphing and scientific calculators. - -Unless explicitly noted, these functions support :class:`int`, -:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Behaviour with other types (whether in the numeric tower or not) is -currently unsupported. Collections with a mix of types are also undefined -and implementation-dependent. If your input data consists of mixed types, -you may be able to use :func:`map` to ensure a consistent result, for -example: ``map(float, input_data)``. - -Averages and measures of central location ------------------------------------------ - -These functions calculate an average or typical value from a population -or sample. - -======================= =============================================================== -:func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean. -:func:`geometric_mean` Geometric mean of data. -:func:`harmonic_mean` Harmonic mean of data. -:func:`median` Median (middle value) of data. -:func:`median_low` Low median of data. -:func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. -:func:`mode` Single mode (most common value) of discrete or nominal data. -:func:`multimode` List of modes (most common values) of discrete or nomimal data. -:func:`quantiles` Divide data into intervals with equal probability. -======================= =============================================================== - -Measures of spread ------------------- - -These functions calculate a measure of how much the population or sample -tends to deviate from the typical or average values. - -======================= ============================================= -:func:`pstdev` Population standard deviation of data. -:func:`pvariance` Population variance of data. -:func:`stdev` Sample standard deviation of data. -:func:`variance` Sample variance of data. -======================= ============================================= - - -Function details ----------------- - -Note: The functions do not require the data given to them to be sorted. -However, for reading convenience, most of the examples show sorted sequences. - -.. function:: mean(data) - - Return the sample arithmetic mean of *data* which can be a sequence or iterable. - - The arithmetic mean is the sum of the data divided by the number of data - points. It is commonly called "the average", although it is only one of many - different mathematical averages. It is a measure of the central location of - the data. - - If *data* is empty, :exc:`StatisticsError` will be raised. - - Some examples of use: - - .. doctest:: - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - >>> mean([-1.0, 2.5, 3.25, 5.75]) - 2.625 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - .. note:: - - The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of - the data points. For more robust measures of central location, see - :func:`median` and :func:`mode`. - - The sample mean gives an unbiased estimate of the true population mean, - so that when taken on average over all the possible samples, - ``mean(sample)`` converges on the true mean of the entire population. If - *data* represents the entire population rather than a sample, then - ``mean(data)`` is equivalent to calculating the true population mean μ. - - -.. function:: fmean(data) - - Convert *data* to floats and compute the arithmetic mean. - - This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The *data* may be a sequence or iterable. If the input - dataset is empty, raises a :exc:`StatisticsError`. - - .. doctest:: - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - - .. versionadded:: 3.8 - - -.. function:: geometric_mean(data) - - Convert *data* to floats and compute the geometric mean. - - The geometric mean indicates the central tendency or typical value of the - *data* using the product of the values (as opposed to the arithmetic mean - which uses their sum). - - Raises a :exc:`StatisticsError` if the input dataset is empty, - if it contains a zero, or if it contains a negative value. - The *data* may be a sequence or iterable. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - .. doctest:: - - >>> round(geometric_mean([54, 24, 36]), 1) - 36.0 - - .. versionadded:: 3.8 - - -.. function:: harmonic_mean(data, weights=None) - - Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then - equal weighting is assumed. - - The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the - reciprocals of the data. For example, the harmonic mean of three values *a*, - *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the - values is zero, the result will be zero. - - The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging - ratios or rates, for example speeds. - - Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. - What is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - :exc:`StatisticsError` is raised if *data* is empty, any element - is less than zero, or if the weighted sum isn't positive. - - The current algorithm has an early-out when it encounters a zero - in the input. This means that the subsequent inputs are not tested - for validity. (This behavior may change in the future.) - - .. versionadded:: 3.6 - - .. versionchanged:: 3.10 - Added support for *weights*. - -.. function:: median(data) - - Return the median (middle value) of numeric data, using the common "mean of - middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterable. - - The median is a robust measure of central location and is less affected by - the presence of outliers. When the number of data points is odd, the - middle data point is returned: - - .. doctest:: - - >>> median([1, 3, 5]) - 3 - - When the number of data points is even, the median is interpolated by taking - the average of the two middle values: - - .. doctest:: - - >>> median([1, 3, 5, 7]) - 4.0 - - This is suited for when your data is discrete, and you don't mind that the - median may not be an actual data point. - - If the data is ordinal (supports order operations) but not numeric (doesn't - support addition), consider using :func:`median_low` or :func:`median_high` - instead. - -.. function:: median_low(data) - - Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. - - The low median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the smaller of - the two middle values is returned. - - .. doctest:: - - >>> median_low([1, 3, 5]) - 3 - >>> median_low([1, 3, 5, 7]) - 3 - - Use the low median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_high(data) - - Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - The high median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the larger of - the two middle values is returned. - - .. doctest:: - - >>> median_high([1, 3, 5]) - 3 - >>> median_high([1, 3, 5, 7]) - 5 - - Use the high median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_grouped(data, interval=1) - - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - .. doctest:: - - >>> median_grouped([52, 52, 53, 54]) - 52.5 - - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: - - .. doctest:: - - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: - - .. doctest:: - - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: - - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. - - -.. function:: mode(data) - - Return the single most common data point from discrete or nominal *data*. - The mode (when it exists) is the most typical value and serves as a - measure of central location. - - If there are multiple modes with the same frequency, returns the first one - encountered in the *data*. If the smallest or largest of those is - desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. - If the input *data* is empty, :exc:`StatisticsError` is raised. - - ``mode`` assumes discrete data and returns a single value. This is the - standard treatment of the mode as commonly taught in schools: - - .. doctest:: - - >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) - 3 - - The mode is unique in that it is the only statistic in this package that - also applies to nominal (non-numeric) data: - - .. doctest:: - - >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) - 'red' - - .. versionchanged:: 3.8 - Now handles multimodal datasets by returning the first mode encountered. - Formerly, it raised :exc:`StatisticsError` when more than one mode was - found. - - -.. function:: multimode(data) - - Return a list of the most frequently occurring values in the order they - were first encountered in the *data*. Will return more than one result if - there are multiple modes or an empty list if the *data* is empty: - - .. doctest:: - - >>> multimode('aabbbbccddddeeffffgg') - ['b', 'd', 'f'] - >>> multimode('') - [] - - .. versionadded:: 3.8 - - -.. function:: pstdev(data, mu=None) - - Return the population standard deviation (the square root of the population - variance). See :func:`pvariance` for arguments and other details. - - .. doctest:: - - >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 0.986893273527251 - - -.. function:: pvariance(data, mu=None) - - Return the population variance of *data*, a non-empty sequence or iterable - of real-valued numbers. Variance, or second moment about the mean, is a - measure of the variability (spread or dispersion) of data. A large - variance indicates that the data is spread out; a small variance indicates - it is clustered closely around the mean. - - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), - the arithmetic mean is automatically calculated. - - Use this function to calculate the variance from the entire population. To - estimate the variance from a sample, the :func:`variance` function is usually - a better choice. - - Raises :exc:`StatisticsError` if *data* is empty. - - Examples: - - .. doctest:: - - >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] - >>> pvariance(data) - 1.25 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *mu* to avoid recalculation: - - .. doctest:: - - >>> mu = mean(data) - >>> pvariance(data, mu) - 1.25 - - Decimals and Fractions are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('24.815') - - >>> from fractions import Fraction as F - >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) - Fraction(13, 72) - - .. note:: - - When called with the entire population, this gives the population variance - σ². When called on a sample instead, this is the biased sample variance - s², also known as variance with N degrees of freedom. - - If you somehow know the true population mean μ, you may use this - function to calculate the variance of a sample, giving the known - population mean as the second argument. Provided the data points are a - random sample of the population, the result will be an unbiased estimate - of the population variance. - - -.. function:: stdev(data, xbar=None) - - Return the sample standard deviation (the square root of the sample - variance). See :func:`variance` for arguments and other details. - - .. doctest:: - - >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 1.0810874155219827 - - -.. function:: variance(data, xbar=None) - - Return the sample variance of *data*, an iterable of at least two real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function when your data is a sample from a population. To calculate - the variance from the entire population, see :func:`pvariance`. - - Raises :exc:`StatisticsError` if *data* has fewer than two values. - - Examples: - - .. doctest:: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: - - .. doctest:: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not attempt to verify that you have passed the actual mean - as *xbar*. Using arbitrary values for *xbar* can lead to invalid or - impossible results. - - Decimal and Fraction values are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - .. note:: - - This is the sample variance s² with Bessel's correction, also known as - variance with N-1 degrees of freedom. Provided that the data points are - representative (e.g. independent and identically distributed), the result - should be an unbiased estimate of the true population variance. - - If you somehow know the actual population mean μ you should pass it to the - :func:`pvariance` function as the *mu* parameter to get the variance of a - sample. - -.. function:: quantiles(data, *, n=4, method='exclusive') - - Divide *data* into *n* continuous intervals with equal probability. - Returns a list of ``n - 1`` cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set - *n* to 100 for percentiles which gives the 99 cuts points that separate - *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* - is not least 1. - - The *data* can be any iterable containing sample data. For meaningful - results, the number of data points in *data* should be larger than *n*. - Raises :exc:`StatisticsError` if there are not at least two data points. - - The cut points are linearly interpolated from the - two nearest data points. For example, if a cut point falls one-third - of the distance between two sample values, ``100`` and ``112``, the - cut-point will evaluate to ``104``. - - The *method* for computing quantiles can be varied depending on - whether the *data* includes or excludes the lowest and - highest possible values from the population. - - The default *method* is "exclusive" and is used for data sampled from - a population that can have more extreme values than found in the - samples. The portion of the population falling below the *i-th* of - *m* sorted data points is computed as ``i / (m + 1)``. Given nine - sample values, the method sorts them and assigns the following - percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. - - Setting the *method* to "inclusive" is used for describing population - data or for samples that are known to include the most extreme values - from the population. The minimum value in *data* is treated as the 0th - percentile and the maximum value is treated as the 100th percentile. - The portion of the population falling below the *i-th* of *m* sorted - data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample - values, the method sorts them and assigns the following percentiles: - 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. - - .. doctest:: - - # Decile cut points for empirically sampled data - >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, - ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, - ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, - ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, - ... 103, 107, 101, 81, 109, 104] - >>> [round(q, 1) for q in quantiles(data, n=10)] - [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] - - .. versionadded:: 3.8 - - -Exceptions ----------- - -A single exception is defined: - -.. exception:: StatisticsError - - Subclass of :exc:`ValueError` for statistics-related exceptions. - - -:class:`NormalDist` objects ---------------------------- - -:class:`NormalDist` is a tool for creating and manipulating normal -distributions of a `random variable -`_. It is a -class that treats the mean and standard deviation of data -measurements as a single entity. - -Normal distributions arise from the `Central Limit Theorem -`_ and have a wide range -of applications in statistics. - -.. class:: NormalDist(mu=0.0, sigma=1.0) - - Returns a new *NormalDist* object where *mu* represents the `arithmetic - mean `_ and *sigma* - represents the `standard deviation - `_. - - If *sigma* is negative, raises :exc:`StatisticsError`. - - .. attribute:: mean - - A read-only property for the `arithmetic mean - `_ of a normal - distribution. - - .. attribute:: median - - A read-only property for the `median - `_ of a normal - distribution. - - .. attribute:: mode - - A read-only property for the `mode - `_ of a normal - distribution. - - .. attribute:: stdev - - A read-only property for the `standard deviation - `_ of a normal - distribution. - - .. attribute:: variance - - A read-only property for the `variance - `_ of a normal - distribution. Equal to the square of the standard deviation. - - .. classmethod:: NormalDist.from_samples(data) - - Makes a normal distribution instance with *mu* and *sigma* parameters - estimated from the *data* using :func:`fmean` and :func:`stdev`. - - The *data* can be any :term:`iterable` and should consist of values - that can be converted to type :class:`float`. If *data* does not - contain at least two elements, raises :exc:`StatisticsError` because it - takes at least one point to estimate a central value and at least two - points to estimate dispersion. - - .. method:: NormalDist.samples(n, *, seed=None) - - Generates *n* random samples for a given mean and standard deviation. - Returns a :class:`list` of :class:`float` values. - - If *seed* is given, creates a new instance of the underlying random - number generator. This is useful for creating reproducible results, - even in a multi-threading context. - - .. method:: NormalDist.pdf(x) - - Using a `probability density function (pdf) - `_, compute - the relative likelihood that a random variable *X* will be near the - given value *x*. Mathematically, it is the limit of the ratio ``P(x <= - X < x+dx) / dx`` as *dx* approaches zero. - - The relative likelihood is computed as the probability of a sample - occurring in a narrow range divided by the width of the range (hence - the word "density"). Since the likelihood is relative to other points, - its value can be greater than `1.0`. - - .. method:: NormalDist.cdf(x) - - Using a `cumulative distribution function (cdf) - `_, - compute the probability that a random variable *X* will be less than or - equal to *x*. Mathematically, it is written ``P(X <= x)``. - - .. method:: NormalDist.inv_cdf(p) - - Compute the inverse cumulative distribution function, also known as the - `quantile function `_ - or the `percent-point - `_ - function. Mathematically, it is written ``x : P(X <= x) = p``. - - Finds the value *x* of the random variable *X* such that the - probability of the variable being less than or equal to that value - equals the given probability *p*. - - .. method:: NormalDist.overlap(other) - - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving `the overlapping area for - the two probability density functions - `_. - - .. method:: NormalDist.quantiles(n=4) - - Divide the normal distribution into *n* continuous intervals with - equal probability. Returns a list of (n - 1) cut points separating - the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution into 100 equal sized groups. - - .. method:: NormalDist.zscore(x) - - Compute the - `Standard Score `_ - describing *x* in terms of the number of standard deviations - above or below the mean of the normal distribution: - ``(x - mean) / stdev``. - - .. versionadded:: 3.9 - - Instances of :class:`NormalDist` support addition, subtraction, - multiplication and division by a constant. These operations - are used for translation and scaling. For example: - - .. doctest:: - - >>> temperature_february = NormalDist(5, 2.5) # Celsius - >>> temperature_february * (9/5) + 32 # Fahrenheit - NormalDist(mu=41.0, sigma=4.5) - - Dividing a constant by an instance of :class:`NormalDist` is not supported - because the result wouldn't be normally distributed. - - Since normal distributions arise from additive effects of independent - variables, it is possible to `add and subtract two independent normally - distributed random variables - `_ - represented as instances of :class:`NormalDist`. For example: - - .. doctest:: - - >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) - >>> drug_effects = NormalDist(0.4, 0.15) - >>> combined = birth_weights + drug_effects - >>> round(combined.mean, 1) - 3.1 - >>> round(combined.stdev, 1) - 0.5 - - .. versionadded:: 3.8 - - -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`NormalDist` readily solves classic probability problems. - -For example, given `historical data for SAT exams -`_ showing -that scores are normally distributed with a mean of 1060 and a standard -deviation of 195, determine the percentage of students with test scores -between 1100 and 1200, after rounding to the nearest whole number: - -.. doctest:: - - >>> sat = NormalDist(1060, 195) - >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) - >>> round(fraction * 100.0, 1) - 18.4 - -Find the `quartiles `_ and `deciles -`_ for the SAT scores: - -.. doctest:: - - >>> list(map(round, sat.quantiles())) - [928, 1060, 1192] - >>> list(map(round, sat.quantiles(n=10))) - [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] - -To estimate the distribution for a model than isn't easy to solve -analytically, :class:`NormalDist` can generate input samples for a `Monte -Carlo simulation `_: - -.. doctest:: - - >>> def model(x, y, z): - ... return (3*x + 7*x*y - 5*y) / (11 * z) - ... - >>> n = 100_000 - >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) - >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) - >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) - >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP - [1.4591308524824727, 1.8035946855390597, 2.175091447274739] - -Normal distributions can be used to approximate `Binomial -distributions `_ -when the sample size is large and when the probability of a successful -trial is near 50%. - -For example, an open source conference has 750 attendees and two rooms with a -500 person capacity. There is a talk about Python and another about Ruby. -In previous conferences, 65% of the attendees preferred to listen to Python -talks. Assuming the population preferences haven't changed, what is the -probability that the Python room will stay within its capacity limits? - -.. doctest:: - - >>> n = 750 # Sample size - >>> p = 0.65 # Preference for Python - >>> q = 1.0 - p # Preference for Ruby - >>> k = 500 # Room capacity - - >>> # Approximation using the cumulative normal distribution - >>> from math import sqrt - >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) - 0.8402 - - >>> # Solution using the cumulative binomial distribution - >>> from math import comb, fsum - >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) - 0.8402 - - >>> # Approximation using a simulation - >>> from random import seed, choices - >>> seed(8675309) - >>> def trial(): - ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') - >>> mean(trial() <= k for i in range(10_000)) - 0.8398 - -Normal distributions commonly arise in machine learning problems. - -Wikipedia has a `nice example of a Naive Bayesian Classifier -`_. -The challenge is to predict a person's gender from measurements of normally -distributed features including height, weight, and foot size. - -We're given a training dataset with measurements for eight people. The -measurements are assumed to be normally distributed, so we summarize the data -with :class:`NormalDist`: - -.. doctest:: - - >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) - >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) - >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) - >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) - >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) - >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) - -Next, we encounter a new person whose feature measurements are known but whose -gender is unknown: - -.. doctest:: - - >>> ht = 6.0 # height - >>> wt = 130 # weight - >>> fs = 8 # foot size - -Starting with a 50% `prior probability -`_ of being male or female, -we compute the posterior as the prior times the product of likelihoods for the -feature measurements given the gender: - -.. doctest:: - - >>> prior_male = 0.5 - >>> prior_female = 0.5 - >>> posterior_male = (prior_male * height_male.pdf(ht) * - ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) - - >>> posterior_female = (prior_female * height_female.pdf(ht) * - ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) - -The final prediction goes to the largest posterior. This is known as the -`maximum a posteriori -`_ or MAP: - -.. doctest:: - - >>> 'male' if posterior_male > posterior_female else 'female' - 'female' - - -.. - # This modelines must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/tempfile.rst.bak b/Doc/library/tempfile.rst.bak deleted file mode 100644 index 2b8a35e2651e731..000000000000000 --- a/Doc/library/tempfile.rst.bak +++ /dev/null @@ -1,374 +0,0 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ - -.. module:: tempfile - :synopsis: Generate temporary files and directories. - -.. sectionauthor:: Zack Weinberg - -**Source code:** :source:`Lib/tempfile.py` - -.. index:: - pair: temporary; file name - pair: temporary; file - --------------- - -This module creates temporary files and directories. It works on all -supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. - -All the user-callable functions and constructors take additional arguments which -allow direct control over the location and name of temporary files and -directories. Files names used by this module include a string of -random characters which allows those files to be securely created in -shared temporary directories. -To maintain backward compatibility, the argument order is somewhat odd; it -is recommended to use keyword arguments for clarity. - -The module defines the following user-callable items: - -.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - Return a :term:`file-like object` that can be used as a temporary storage area. - The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon - as it is closed (including an implicit close when the object is garbage - collected). Under Unix, the directory entry for the file is either not created at all or is removed - immediately after the file is created. Other platforms do not support - this; your code should not rely on a temporary file created using this - function having or not having a visible name in the file system. - - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or - destruction of the file object the temporary file will be removed - from the filesystem. - - The *mode* parameter defaults to ``'w+b'`` so that the file created can - be read and written without being closed. Binary mode is used so that it - behaves consistently on all platforms without regard for the data that is - stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for - :func:`open`. - - The *dir*, *prefix* and *suffix* parameters have the same meaning and - defaults as with :func:`mkstemp`. - - The returned object is a true file object on POSIX platforms. On other - platforms, it is a file-like object whose :attr:`!file` attribute is the - underlying true file object. - - The :py:data:`os.O_TMPFILE` flag is used if it is available and works - (Linux-specific, requires Linux kernel 3.11 or later). - - .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile - - .. versionchanged:: 3.5 - - The :py:data:`os.O_TMPFILE` flag is now used if available. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - the file is guaranteed to have a visible name in the file system (on - Unix, the directory entry is not unlinked). That name can be retrieved - from the :attr:`name` attribute of the returned - file-like object. Whether the name can be - used to open the file a second time, while the named temporary file is - still open, varies across platforms (it can be so used on Unix; it cannot - on Windows NT or later). If *delete* is true (the default), the file is - deleted as soon as it is closed. - The returned object is always a file-like object whose :attr:`!file` - attribute is the underlying true file object. This file-like object can - be used in a :keyword:`with` statement, just like a normal file. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the - contents are written to disk and operation proceeds as with - :func:`TemporaryFile`. - - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. - - The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object - (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This - file-like object can be used in a :keyword:`with` statement, just like - a normal file. - - .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) - - This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory - and all its contents are removed from the filesystem. - - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory - - .. versionadded:: 3.2 - - -.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) - - Creates a temporary file in the most secure manner possible. There are - no race conditions in the file's creation, assuming that the platform - properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The - file is readable and writable only by the creating user ID. If the - platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. - - Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible - for deleting the temporary file when done with it. - - If *suffix* is not ``None``, the file name will end with that suffix, - otherwise there will be no suffix. :func:`mkstemp` does not put a dot - between the file name and the suffix; if you need one, put it at the - beginning of *suffix*. - - If *prefix* is not ``None``, the file name will begin with that prefix; - otherwise, a default prefix is used. The default is the return value of - :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. - - If *dir* is not ``None``, the file will be created in that directory; - otherwise, a default directory is used. The default directory is chosen - from a platform-dependent list, but the user of the application can - control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* - environment variables. There is thus no guarantee that the generated - filename will have any nice properties, such as not requiring quoting - when passed to external commands via ``os.popen()``. - - If any of *suffix*, *prefix*, and *dir* are not - ``None``, they must be the same type. - If they are bytes, the returned name will be bytes instead of str. - If you want to force a bytes return value with otherwise default behavior, - pass ``suffix=b''``. - - If *text* is specified and true, the file is opened in text mode. - Otherwise, (the default) the file is opened in binary mode. - - :func:`mkstemp` returns a tuple containing an OS-level handle to an open - file (as would be returned by :func:`os.open`) and the absolute pathname - of that file, in that order. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: mkdtemp(suffix=None, prefix=None, dir=None) - - Creates a temporary directory in the most secure manner possible. There - are no race conditions in the directory's creation. The directory is - readable, writable, and searchable only by the creating user ID. - - The user of :func:`mkdtemp` is responsible for deleting the temporary - directory and its contents when done with it. - - The *prefix*, *suffix*, and *dir* arguments are the same as for - :func:`mkstemp`. - - :func:`mkdtemp` returns the absolute pathname of the new directory. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: gettempdir() - - Return the name of the directory used for temporary files. This - defines the default value for the *dir* argument to all functions - in this module. - - Python searches a standard list of directories to find one which - the calling user can create files in. The list is: - - #. The directory named by the :envvar:`TMPDIR` environment variable. - - #. The directory named by the :envvar:`TEMP` environment variable. - - #. The directory named by the :envvar:`TMP` environment variable. - - #. A platform-specific location: - - * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, - :file:`\\TEMP`, and :file:`\\TMP`, in that order. - - * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and - :file:`/usr/tmp`, in that order. - - #. As a last resort, the current working directory. - - The result of this search is cached, see the description of - :data:`tempdir` below. - - .. versionchanged:: 3.10 - - Always returns a str. Previously it would return any :data:`tempdir` - value regardless of type so long as it was not ``None``. - -.. function:: gettempdirb() - - Same as :func:`gettempdir` but the return value is in bytes. - - .. versionadded:: 3.5 - -.. function:: gettempprefix() - - Return the filename prefix used to create temporary files. This does not - contain the directory component. - -.. function:: gettempprefixb() - - Same as :func:`gettempprefix` but the return value is in bytes. - - .. versionadded:: 3.5 - -The module uses a global variable to store the name of the directory -used for temporary files returned by :func:`gettempdir`. It can be -set directly to override the selection process, but this is discouraged. -All functions in this module take a *dir* argument which can be used -to specify the directory. This is the recommended approach that does -not surprise other unsuspecting code by changing global API behavior. - -.. data:: tempdir - - When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to the functions defined in this - module, including its type, bytes or str. It cannot be a - :term:`path-like object`. - - If ``tempdir`` is ``None`` (the default) at any call to any of the above - functions except :func:`gettempprefix` it is initialized following the - algorithm described in :func:`gettempdir`. - - .. note:: - - Beware that if you set ``tempdir`` to a bytes value, there is a - nasty side effect: The global default return type of - :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no - explicit ``prefix``, ``suffix``, or ``dir`` arguments of type - str are supplied. Please do not write code expecting or - depending on this. This awkward behavior is maintained for - compatibility with the historcal implementation. - -.. _tempfile-examples: - -Examples --------- - -Here are some examples of typical usage of the :mod:`tempfile` module:: - - >>> import tempfile - - # create a temporary file and write some data to it - >>> fp = tempfile.TemporaryFile() - >>> fp.write(b'Hello world!') - # read data from file - >>> fp.seek(0) - >>> fp.read() - b'Hello world!' - # close the file, it will be removed - >>> fp.close() - - # create a temporary file using a context manager - >>> with tempfile.TemporaryFile() as fp: - ... fp.write(b'Hello world!') - ... fp.seek(0) - ... fp.read() - b'Hello world!' - >>> - # file is now closed and removed - - # create a temporary directory using the context manager - >>> with tempfile.TemporaryDirectory() as tmpdirname: - ... print('created temporary directory', tmpdirname) - >>> - # directory and contents have been removed - - -Deprecated functions and variables ----------------------------------- - -A historical way to create temporary files was to first generate a -file name with the :func:`mktemp` function and then create a file -using this name. Unfortunately this is not secure, because a different -process may create a file with this name in the time between the call -to :func:`mktemp` and the subsequent attempt to create the file by the -first process. The solution is to combine the two steps and create the -file immediately. This approach is used by :func:`mkstemp` and the -other functions described above. - -.. function:: mktemp(suffix='', prefix='tmp', dir=None) - - .. deprecated:: 2.3 - Use :func:`mkstemp` instead. - - Return an absolute pathname of a file that did not exist at the time the - call is made. The *prefix*, *suffix*, and *dir* arguments are similar - to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` - and ``prefix=None`` are not supported. - - .. warning:: - - Use of this function may introduce a security hole in your program. By - the time you get around to doing anything with the file name it returns, - someone else may have beaten you to the punch. :func:`mktemp` usage can - be replaced easily with :func:`NamedTemporaryFile`, passing it the - ``delete=False`` parameter:: - - >>> f = NamedTemporaryFile(delete=False) - >>> f.name - '/tmp/tmptjujjt' - >>> f.write(b"Hello World!\n") - 13 - >>> f.close() - >>> os.unlink(f.name) - >>> os.path.exists(f.name) - False diff --git a/Doc/library/venv.rst.bak b/Doc/library/venv.rst.bak deleted file mode 100644 index 5d4a36481f1dcc5..000000000000000 --- a/Doc/library/venv.rst.bak +++ /dev/null @@ -1,496 +0,0 @@ -:mod:`venv` --- Creation of virtual environments -================================================ - -.. module:: venv - :synopsis: Creation of virtual environments. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/venv/` - -.. index:: pair: Environments; virtual - --------------- - -The :mod:`venv` module provides support for creating lightweight "virtual -environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (which -matches the version of the binary that was used to create this environment) and -can have its own independent set of installed Python packages in its site -directories. - -See :pep:`405` for more information about Python virtual environments. - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - - -Creating virtual environments ------------------------------ - -.. include:: /using/venv-create.inc - - -.. _venv-def: - -.. note:: A virtual environment is a Python environment such that the Python - interpreter, libraries and scripts installed into it are isolated from those - installed in other virtual environments, and (by default) any libraries - installed in a "system" Python, i.e., one which is installed as part of your - operating system. - - A virtual environment is a directory tree which contains Python executable - files and other files which indicate that it is a virtual environment. - - Common installation tools such as setuptools_ and pip_ work as - expected with virtual environments. In other words, when a virtual - environment is active, they install Python packages into the virtual - environment without needing to be told to do so explicitly. - - When a virtual environment is active (i.e., the virtual environment's Python - interpreter is running), the attributes :attr:`sys.prefix` and - :attr:`sys.exec_prefix` point to the base directory of the virtual - environment, whereas :attr:`sys.base_prefix` and - :attr:`sys.base_exec_prefix` point to the non-virtual environment Python - installation which was used to create the virtual environment. If a virtual - environment is not active, then :attr:`sys.prefix` is the same as - :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as - :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment - Python installation). - - When a virtual environment is active, any options that change the - installation path will be ignored from all :mod:`distutils` configuration - files to prevent projects being inadvertently installed outside of the - virtual environment. - - When working in a command shell, users can make a virtual environment active - by running an ``activate`` script in the virtual environment's executables - directory (the precise filename and command to use the file is - shell-dependent), which prepends the virtual environment's directory for - executables to the ``PATH`` environment variable for the running shell. There - should be no need in other circumstances to activate a virtual - environment; scripts installed into virtual environments have a "shebang" - line which points to the virtual environment's Python interpreter. This means - that the script will run with that interpreter regardless of the value of - ``PATH``. On Windows, "shebang" line processing is supported if you have the - Python Launcher for Windows installed (this was added to Python in 3.3 - see - :pep:`397` for more details). Thus, double-clicking an installed script in a - Windows Explorer window should run the script with the correct interpreter - without there needing to be any reference to its virtual environment in - ``PATH``. - - -.. _venv-api: - -API ---- - -.. highlight:: python - -The high-level method described above makes use of a simple API which provides -mechanisms for third-party virtual environment creators to customize environment -creation according to their needs, the :class:`EnvBuilder` class. - -.. class:: EnvBuilder(system_site_packages=False, clear=False, \ - symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False) - - The :class:`EnvBuilder` class accepts the following keyword arguments on - instantiation: - - * ``system_site_packages`` -- a Boolean value indicating that the system Python - site-packages should be available to the environment (defaults to ``False``). - - * ``clear`` -- a Boolean value which, if true, will delete the contents of - any existing target directory, before creating the environment. - - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary rather than copying. - - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing - environment with the running Python - for use when that Python has been - upgraded in-place (defaults to ``False``). - - * ``with_pip`` -- a Boolean value which, if true, ensures pip is - installed in the virtual environment. This uses :mod:`ensurepip` with - the ``--default-pip`` option. - - * ``prompt`` -- a String to be used after virtual environment is activated - (defaults to ``None`` which means directory name of the environment would - be used). If the special string ``"."`` is provided, the basename of the - current directory is used as the prompt. - - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionadded:: 3.6 - Added the ``prompt`` parameter - - .. versionadded:: 3.9 - Added the ``upgrade_deps`` parameter - - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: - - .. method:: create(env_dir) - - Create a virtual environment by specifying the target directory - (absolute or relative to the current directory) which is to contain the - virtual environment. The ``create`` method will either create the - environment in the specified directory, or raise an appropriate - exception. - - The ``create`` method of the :class:`EnvBuilder` class illustrates the - hooks available for subclass customization:: - - def create(self, env_dir): - """ - Create a virtualized Python environment in a directory. - env_dir is the target directory to create an environment in. - """ - env_dir = os.path.abspath(env_dir) - context = self.ensure_directories(env_dir) - self.create_configuration(context) - self.setup_python(context) - self.setup_scripts(context) - self.post_setup(context) - - Each of the methods :meth:`ensure_directories`, - :meth:`create_configuration`, :meth:`setup_python`, - :meth:`setup_scripts` and :meth:`post_setup` can be overridden. - - .. method:: ensure_directories(env_dir) - - Creates the environment directory and all necessary directories, and - returns a context object. This is just a holder for attributes (such as - paths), for use by the other methods. The directories are allowed to - exist already, as long as either ``clear`` or ``upgrade`` were - specified to allow operating on an existing environment directory. - - .. method:: create_configuration(context) - - Creates the ``pyvenv.cfg`` configuration file in the environment. - - .. method:: setup_python(context) - - Creates a copy or symlink to the Python executable in the environment. - On POSIX systems, if a specific executable ``python3.x`` was used, - symlinks to ``python`` and ``python3`` will be created pointing to that - executable, unless files with those names already exist. - - .. method:: setup_scripts(context) - - Installs activation scripts appropriate to the platform into the virtual - environment. - - .. method:: upgrade_dependencies(context) - - Upgrades the core venv dependency packages (currently ``pip`` and - ``setuptools``) in the environment. This is done by shelling out to the - ``pip`` executable in the environment. - - .. versionadded:: 3.9 - - .. method:: post_setup(context) - - A placeholder method which can be overridden in third party - implementations to pre-install packages in the virtual environment or - perform other post-creation steps. - - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - - .. method:: install_scripts(context, path) - - *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the - directory corresponding to :data:`os.name` are copied after some text - replacement of placeholders: - - * ``__VENV_DIR__`` is replaced with the absolute path of the environment - directory. - - * ``__VENV_NAME__`` is replaced with the environment name (final path - segment of environment directory). - - * ``__VENV_PROMPT__`` is replaced with the prompt (the environment - name surrounded by parentheses and with a following space) - - * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory - (either ``bin`` or ``Scripts``). - - * ``__VENV_PYTHON__`` is replaced with the absolute path of the - environment's executable. - - The directories are allowed to exist (for when an existing environment - is being upgraded). - -There is also a module-level convenience function: - -.. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False) - - Create an :class:`EnvBuilder` with the given keyword arguments, and call its - :meth:`~EnvBuilder.create` method with the *env_dir* argument. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionchanged:: 3.6 - Added the ``prompt`` parameter - - .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter - -An example of extending ``EnvBuilder`` --------------------------------------- - -The following script shows how to extend :class:`EnvBuilder` by implementing a -subclass which installs setuptools and pip into a created virtual environment:: - - import os - import os.path - from subprocess import Popen, PIPE - import sys - from threading import Thread - from urllib.parse import urlparse - from urllib.request import urlretrieve - import venv - - class ExtendedEnvBuilder(venv.EnvBuilder): - """ - This builder installs setuptools and pip so that you can pip or - easy_install other packages into the created virtual environment. - - :param nodist: If true, setuptools and pip are not installed into the - created virtual environment. - :param nopip: If true, pip is not installed into the created - virtual environment. - :param progress: If setuptools or pip are installed, the progress of the - installation can be monitored by passing a progress - callable. If specified, it is called with two - arguments: a string indicating some progress, and a - context indicating where the string is coming from. - The context argument can have one of three values: - 'main', indicating that it is called from virtualize() - itself, and 'stdout' and 'stderr', which are obtained - by reading lines from the output streams of a subprocess - which is used to install the app. - - If a callable is not specified, default progress - information is output to sys.stderr. - """ - - def __init__(self, *args, **kwargs): - self.nodist = kwargs.pop('nodist', False) - self.nopip = kwargs.pop('nopip', False) - self.progress = kwargs.pop('progress', None) - self.verbose = kwargs.pop('verbose', False) - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ - Set up any packages which need to be pre-installed into the - virtual environment being created. - - :param context: The information for the virtual environment - creation request being processed. - """ - os.environ['VIRTUAL_ENV'] = context.env_dir - if not self.nodist: - self.install_setuptools(context) - # Can't install pip without setuptools - if not self.nopip and not self.nodist: - self.install_pip(context) - - def reader(self, stream, context): - """ - Read lines from a subprocess' output stream and either pass to a progress - callable (if specified) or write progress information to sys.stderr. - """ - progress = self.progress - while True: - s = stream.readline() - if not s: - break - if progress is not None: - progress(s, context) - else: - if not self.verbose: - sys.stderr.write('.') - else: - sys.stderr.write(s.decode('utf-8')) - sys.stderr.flush() - stream.close() - - def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) - fn = os.path.split(path)[-1] - binpath = context.bin_path - distpath = os.path.join(binpath, fn) - # Download script into the virtual environment's binaries folder - urlretrieve(url, distpath) - progress = self.progress - if self.verbose: - term = '\n' - else: - term = '' - if progress is not None: - progress('Installing %s ...%s' % (name, term), 'main') - else: - sys.stderr.write('Installing %s ...%s' % (name, term)) - sys.stderr.flush() - # Install in the virtual environment - args = [context.env_exe, fn] - p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) - t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) - t1.start() - t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) - t2.start() - p.wait() - t1.join() - t2.join() - if progress is not None: - progress('done.', 'main') - else: - sys.stderr.write('done.\n') - # Clean up - no longer needed - os.unlink(distpath) - - def install_setuptools(self, context): - """ - Install setuptools in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' - self.install_script(context, 'setuptools', url) - # clear up the setuptools archive which gets downloaded - pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') - files = filter(pred, os.listdir(context.bin_path)) - for f in files: - f = os.path.join(context.bin_path, f) - os.unlink(f) - - def install_pip(self, context): - """ - Install pip in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' - self.install_script(context, 'pip', url) - - def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') - else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) - - if __name__ == '__main__': - rc = 1 - try: - main() - rc = 0 - except Exception as e: - print('Error: %s' % e, file=sys.stderr) - sys.exit(rc) - - -This script is also available for download `online -`_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/xml.sax.handler.rst.bak b/Doc/library/xml.sax.handler.rst.bak deleted file mode 100644 index 3746a58c9b9558a..000000000000000 --- a/Doc/library/xml.sax.handler.rst.bak +++ /dev/null @@ -1,463 +0,0 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== - -.. module:: xml.sax.handler - :synopsis: Base classes for SAX event handlers. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/handler.py` - --------------- - -The SAX API defines five kinds of handlers: content handlers, DTD handlers, -error handlers, entity resolvers and lexical handlers. Applications normally -only need to implement those interfaces whose events they are interested in; -they can implement the interfaces in a single object or in multiple objects. -Handler implementations should inherit from the base classes provided in the -module :mod:`xml.sax.handler`, so that all methods get default implementations. - - -.. class:: ContentHandler - - This is the main callback interface in SAX, and the one most important to - applications. The order of events in this interface mirrors the order of the - information in the document. - - -.. class:: DTDHandler - - Handle DTD events. - - This interface specifies only those DTD events required for basic parsing - (unparsed entities and attributes). - - -.. class:: EntityResolver - - Basic interface for resolving entities. If you create an object implementing - this interface, then register the object with your Parser, the parser will call - the method in your object to resolve all external entities. - - -.. class:: ErrorHandler - - Interface used by the parser to present error and warning messages to the - application. The methods of this object control whether errors are immediately - converted to exceptions or are handled in some other way. - - -.. class:: LexicalHandler - - Interface used by the parser to represent low freqency events which may not - be of interest to many applications. - -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants -for the feature and property names. - - -.. data:: feature_namespaces - - | value: ``"http://xml.org/sax/features/namespaces"`` - | true: Perform Namespace processing. - | false: Optionally do not perform Namespace processing (implies - namespace-prefixes; default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_namespace_prefixes - - | value: ``"http://xml.org/sax/features/namespace-prefixes"`` - | true: Report the original prefixed names and attributes used for Namespace - declarations. - | false: Do not report attributes used for Namespace declarations, and - optionally do not report original prefixed names (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_string_interning - - | value: ``"http://xml.org/sax/features/string-interning"`` - | true: All element names, prefixes, attribute names, Namespace URIs, and - local names are interned using the built-in intern function. - | false: Names are not necessarily interned, although they may be (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_validation - - | value: ``"http://xml.org/sax/features/validation"`` - | true: Report all validation errors (implies external-general-entities and - external-parameter-entities). - | false: Do not report validation errors. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_ges - - | value: ``"http://xml.org/sax/features/external-general-entities"`` - | true: Include all external general (text) entities. - | false: Do not include external general entities. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_pes - - | value: ``"http://xml.org/sax/features/external-parameter-entities"`` - | true: Include all external parameter entities, including the external DTD - subset. - | false: Do not include any external parameter entities, even the external - DTD subset. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: all_features - - List of all features. - - -.. data:: property_lexical_handler - - | value: ``"http://xml.org/sax/properties/lexical-handler"`` - | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) - | description: An optional extension handler for lexical events like - comments. - | access: read/write - - -.. data:: property_declaration_handler - - | value: ``"http://xml.org/sax/properties/declaration-handler"`` - | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) - | description: An optional extension handler for DTD-related events other - than notations and unparsed entities. - | access: read/write - - -.. data:: property_dom_node - - | value: ``"http://xml.org/sax/properties/dom-node"`` - | data type: org.w3c.dom.Node (not supported in Python 2) - | description: When parsing, the current DOM node being visited if this is - a DOM iterator; when not parsing, the root DOM node for iteration. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: property_xml_string - - | value: ``"http://xml.org/sax/properties/xml-string"`` - | data type: String - | description: The literal string of characters that was the source for the - current event. - | access: read-only - - -.. data:: all_properties - - List of all known property names. - - -.. _content-handler-objects: - -ContentHandler Objects ----------------------- - -Users are expected to subclass :class:`ContentHandler` to support their -application. The following methods are called by the parser on the appropriate -events in the input document: - - -.. method:: ContentHandler.setDocumentLocator(locator) - - Called by the parser to give the application a locator for locating the origin - of document events. - - SAX parsers are strongly encouraged (though not absolutely required) to supply a - locator: if it does so, it must supply the locator to the application by - invoking this method before invoking any of the other methods in the - DocumentHandler interface. - - The locator allows the application to determine the end position of any - document-related event, even if the parser is not reporting an error. Typically, - the application will use this information for reporting its own errors (such as - character content that does not match an application's business rules). The - information returned by the locator is probably not sufficient for use with a - search engine. - - Note that the locator will return correct information only during the invocation - of the events in this interface. The application should not attempt to use it at - any other time. - - -.. method:: ContentHandler.startDocument() - - Receive notification of the beginning of a document. - - The SAX parser will invoke this method only once, before any other methods in - this interface or in DTDHandler (except for :meth:`setDocumentLocator`). - - -.. method:: ContentHandler.endDocument() - - Receive notification of the end of a document. - - The SAX parser will invoke this method only once, and it will be the last method - invoked during the parse. The parser shall not invoke this method until it has - either abandoned parsing (because of an unrecoverable error) or reached the end - of input. - - -.. method:: ContentHandler.startPrefixMapping(prefix, uri) - - Begin the scope of a prefix-URI Namespace mapping. - - The information from this event is not necessary for normal Namespace - processing: the SAX XML reader will automatically replace prefixes for element - and attribute names when the ``feature_namespaces`` feature is enabled (the - default). - - There are cases, however, when applications need to use prefixes in character - data or in attribute values, where they cannot safely be expanded automatically; - the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the - information to the application to expand prefixes in those contexts itself, if - necessary. - - .. XXX This is not really the default, is it? MvL - - Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not - guaranteed to be properly nested relative to each-other: all - :meth:`startPrefixMapping` events will occur before the corresponding - :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur - after the corresponding :meth:`endElement` event, but their order is not - guaranteed. - - -.. method:: ContentHandler.endPrefixMapping(prefix) - - End the scope of a prefix-URI mapping. - - See :meth:`startPrefixMapping` for details. This event will always occur after - the corresponding :meth:`endElement` event, but the order of - :meth:`endPrefixMapping` events is not otherwise guaranteed. - - -.. method:: ContentHandler.startElement(name, attrs) - - Signals the start of an element in non-namespace mode. - - The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` - interface (see :ref:`attributes-objects`) containing the attributes of - the element. The object passed as *attrs* may be re-used by the parser; holding - on to a reference to it is not a reliable way to keep a copy of the attributes. - To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* - object. - - -.. method:: ContentHandler.endElement(name) - - Signals the end of an element in non-namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElement` event. - - -.. method:: ContentHandler.startElementNS(name, qname, attrs) - - Signals the start of an element in namespace mode. - - The *name* parameter contains the name of the element type as a ``(uri, - localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in - the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) - containing the attributes of the element. If no namespace is associated with - the element, the *uri* component of *name* will be ``None``. The object passed - as *attrs* may be re-used by the parser; holding on to a reference to it is not - a reliable way to keep a copy of the attributes. To keep a copy of the - attributes, use the :meth:`copy` method of the *attrs* object. - - Parsers may set the *qname* parameter to ``None``, unless the - ``feature_namespace_prefixes`` feature is activated. - - -.. method:: ContentHandler.endElementNS(name, qname) - - Signals the end of an element in namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElementNS` method, likewise the *qname* parameter. - - -.. method:: ContentHandler.characters(content) - - Receive notification of character data. - - The Parser will call this method to report each chunk of character data. SAX - parsers may return all contiguous character data in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity so that the Locator provides useful - information. - - *content* may be a string or bytes instance; the ``expat`` reader module - always produces strings. - - .. note:: - - The earlier SAX 1 interface provided by the Python XML Special Interest Group - used a more Java-like interface for this method. Since most parsers used from - Python did not take advantage of the older interface, the simpler signature was - chosen to replace it. To convert old code to the new interface, use *content* - instead of slicing content with the old *offset* and *length* parameters. - - -.. method:: ContentHandler.ignorableWhitespace(whitespace) - - Receive notification of ignorable whitespace in element content. - - Validating Parsers must use this method to report each chunk of ignorable - whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating - parsers may also use this method if they are capable of parsing and using - content models. - - SAX parsers may return all contiguous whitespace in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity, so that the Locator provides useful - information. - - -.. method:: ContentHandler.processingInstruction(target, data) - - Receive notification of a processing instruction. - - The Parser will invoke this method once for each processing instruction found: - note that processing instructions may occur before or after the main document - element. - - A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a - text declaration (XML 1.0, section 4.3.1) using this method. - - -.. method:: ContentHandler.skippedEntity(name) - - Receive notification of a skipped entity. - - The Parser will invoke this method once for each entity skipped. Non-validating - processors may skip entities if they have not seen the declarations (because, - for example, the entity was declared in an external DTD subset). All processors - may skip external entities, depending on the values of the - ``feature_external_ges`` and the ``feature_external_pes`` properties. - - -.. _dtd-handler-objects: - -DTDHandler Objects ------------------- - -:class:`DTDHandler` instances provide the following methods: - - -.. method:: DTDHandler.notationDecl(name, publicId, systemId) - - Handle a notation declaration event. - - -.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) - - Handle an unparsed entity declaration event. - - -.. _entity-resolver-objects: - -EntityResolver Objects ----------------------- - - -.. method:: EntityResolver.resolveEntity(publicId, systemId) - - Resolve the system identifier of an entity and return either the system - identifier to read from as a string, or an InputSource to read from. The default - implementation returns *systemId*. - - -.. _sax-error-handler: - -ErrorHandler Objects --------------------- - -Objects with this interface are used to receive error and warning information -from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that -implements this interface, then register the object with your -:class:`~xml.sax.xmlreader.XMLReader`, the parser -will call the methods in your object to report all warnings and errors. There -are three levels of errors available: warnings, (possibly) recoverable errors, -and unrecoverable errors. All methods take a :exc:`SAXParseException` as the -only parameter. Errors and warnings may be converted to an exception by raising -the passed-in exception object. - - -.. method:: ErrorHandler.error(exception) - - Called when the parser encounters a recoverable error. If this method does not - raise an exception, parsing may continue, but further document information - should not be expected by the application. Allowing the parser to continue may - allow additional errors to be discovered in the input document. - - -.. method:: ErrorHandler.fatalError(exception) - - Called when the parser encounters an error it cannot recover from; parsing is - expected to terminate when this method returns. - - -.. method:: ErrorHandler.warning(exception) - - Called when the parser presents minor warning information to the application. - Parsing is expected to continue when this method returns, and document - information will continue to be passed to the application. Raising an exception - in this method will cause parsing to end. - - -.. _lexical-handler-objects: - -LexicalHandler Objects ----------------------- -Optional SAX2 handler for lexical events. - -This handler is used to obtain lexical information about an XML -document. Lexical information includes information describing the -document encoding used and XML comments embedded in the document, as -well as section boundaries for the DTD and for any CDATA sections. -The lexical handlers are used in the same manner as content handlers. - -Set the LexicalHandler of an XMLReader by using the setProperty method -with the property identifier -``'http://xml.org/sax/properties/lexical-handler'``. - - -.. method:: LexicalHandler.comment(content) - - Reports a comment anywhere in the document (including the DTD and - outside the document element). - -.. method:: LexicalHandler.startDTD(name, public_id, system_id) - - Reports the start of the DTD declarations if the document has an - associated DTD. - -.. method:: LexicalHandler.endDTD() - - Reports the end of DTD declaration. - -.. method:: LexicalHandler.startCDATA() - - Reports the start of a CDATA marked section. - - The contents of the CDATA marked section will be reported through - the characters handler. - -.. method:: LexicalHandler.endCDATA() - - Reports the end of a CDATA marked section. diff --git a/Doc/library/zipimport.rst.bak b/Doc/library/zipimport.rst.bak deleted file mode 100644 index 62e1e47e980e179..000000000000000 --- a/Doc/library/zipimport.rst.bak +++ /dev/null @@ -1,209 +0,0 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== - -.. module:: zipimport - :synopsis: Support for importing Python modules from ZIP archives. - -.. moduleauthor:: Just van Rossum - -**Source code:** :source:`Lib/zipimport.py` - --------------- - -This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used -by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths -to ZIP archives. - -Typically, :data:`sys.path` is a list of directory names as strings. This module -also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. -The ZIP archive can contain a subdirectory structure to support package imports, -and a path within the archive can be specified to only import from a -subdirectory. For example, the path :file:`example.zip/lib/` would only -import from the :file:`lib/` subdirectory within the archive. - -Any files may be present in the ZIP archive, but only files :file:`.py` and -:file:`.pyc` are available for import. ZIP import of dynamic modules -(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains -:file:`.py` files, Python will not attempt to modify the archive by adding the -corresponding :file:`.pyc` file, meaning that if a ZIP archive -doesn't contain :file:`.pyc` files, importing may be rather slow. - -.. versionchanged:: 3.8 - Previously, ZIP archives with an archive comment were not supported. - -.. seealso:: - - `PKZIP Application Note `_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - :pep:`273` - Import Modules from Zip Archives - Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 - follows the specification in :pep:`273`, but uses an implementation written by Just - van Rossum that uses the import hooks described in :pep:`302`. - - :mod:`importlib` - The implementation of the import machinery - Package providing the relevant protocols for all importers to - implement. - - -This module defines an exception: - -.. exception:: ZipImportError - - Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, - so it can be caught as :exc:`ImportError`, too. - - -.. _zipimporter-objects: - -zipimporter Objects -------------------- - -:class:`zipimporter` is the class for importing ZIP files. - -.. class:: zipimporter(archivepath) - - Create a new zipimporter instance. *archivepath* must be a path to a ZIP - file, or to a specific path within a ZIP file. For example, an *archivepath* - of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory - inside the ZIP file :file:`foo/bar.zip` (provided that it exists). - - :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP - archive. - - .. method:: create_module(spec) - - Implementation of :meth:`importlib.abc.Loader.create_module` that returns - :const:`None` to explicitly request the default semantics. - - .. versionadded:: 3.10 - - - .. method:: exec_module(module) - - Implementation of :meth:`importlib.abc.Loader.exec_module`. - - .. versionadded:: 3.10 - - - .. method:: find_loader(fullname, path=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_module(fullname, path=None) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_spec(fullname, target=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. - - .. versionadded:: 3.10 - - - .. method:: get_code(fullname) - - Return the code object for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be imported. - - - .. method:: get_data(pathname) - - Return the data associated with *pathname*. Raise :exc:`OSError` if the - file wasn't found. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - - - .. method:: get_filename(fullname) - - Return the value ``__file__`` would be set to if the specified module - was imported. Raise :exc:`ZipImportError` if the module couldn't be - imported. - - .. versionadded:: 3.1 - - - .. method:: get_source(fullname) - - Return the source code for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found, return - :const:`None` if the archive does contain the module, but has no source - for it. - - - .. method:: is_package(fullname) - - Return ``True`` if the module specified by *fullname* is a package. Raise - :exc:`ZipImportError` if the module couldn't be found. - - - .. method:: load_module(fullname) - - Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. Returns the imported module on success, - raises :exc:`ZipImportError` on failure. - - .. deprecated:: 3.10 - - Use :meth:`exec_module` instead. - - .. attribute:: archive - - The file name of the importer's associated ZIP file, without a possible - subpath. - - - .. attribute:: prefix - - The subpath within the ZIP file where modules are searched. This is the - empty string for zipimporter objects which point to the root of the ZIP - file. - - The :attr:`archive` and :attr:`prefix` attributes, when combined with a - slash, equal the original *archivepath* argument given to the - :class:`zipimporter` constructor. - - -.. _zipimport-examples: - -Examples --------- - -Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. - -.. code-block:: shell-session - - $ unzip -l example.zip - Archive: example.zip - Length Date Time Name - -------- ---- ---- ---- - 8467 11-26-02 22:30 jwzthreading.py - -------- ------- - 8467 1 file - $ ./python - Python 2.3 (#1, Aug 1 2003, 19:54:32) - >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path - >>> import jwzthreading - >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak deleted file mode 100644 index c4c282e5a04eae9..000000000000000 --- a/Doc/whatsnew/3.10.rst.bak +++ /dev/null @@ -1,1347 +0,0 @@ -**************************** - What's New In Python 3.10 -**************************** - -:Release: |release| -:Date: |today| - -.. Rules for maintenance: - - * Anyone can add text to this document. Do not spend very much time - on the wording of your changes, because your text will probably - get rewritten to some degree. - - * The maintainer will go through Misc/NEWS periodically and add - changes; it's therefore more important to add your changes to - Misc/NEWS than to this file. - - * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. Some changes I consider too small - or esoteric to include. If such a change is added to the text, - I'll just remove it. (This is another reason you shouldn't spend - too much time on writing your addition.) - - * If you want to draw your new text to the attention of the - maintainer, add 'XXX' to the beginning of the paragraph or - section. - - * It's OK to just add a fragmentary note about a change. For - example: "XXX Describe the transmogrify() function added to the - socket module." The maintainer will research the change and - write the necessary text. - - * You can comment out your additions if you like, but it's not - necessary (especially when a final release is some months away). - - * Credit the author of a patch or bugfix. Just the name is - sufficient; the e-mail address isn't necessary. - - * It's helpful to add the bug/patch number as a comment: - - XXX Describe the transmogrify() function added to the socket - module. - (Contributed by P.Y. Developer in :issue:`12345`.) - - This saves the maintainer the effort of going through the Mercurial log - when researching a change. - -This article explains the new features in Python 3.10, compared to 3.9. - -For full details, see the :ref:`changelog `. - -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.10 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- Release highlights -============================= - -.. This section singles out the most important changes in Python 3.10. - Brevity is key. - - -.. PEP-sized items next. - - - -New Features -============ - -.. _whatsnew310-pep563: - -Parenthesized context managers ------------------------------- - -Using enclosing parentheses for continuation across multiple lines -in context managers is now supported. This allows formatting a long -collection of context managers in multiple lines in a similar way -as it was previously possible with import statements. For instance, -all these examples are now valid: - -.. code-block:: python - - with (CtxManager() as example): - ... - - with ( - CtxManager1(), - CtxManager2() - ): - ... - - with (CtxManager1() as example, - CtxManager2()): - ... - - with (CtxManager1(), - CtxManager2() as example): - ... - - with ( - CtxManager1() as example1, - CtxManager2() as example2 - ): - ... - -it is also possible to use a trailing comma at the end of the -enclosed group: - -.. code-block:: python - - with ( - CtxManager1() as example1, - CtxManager2() as example2, - CtxManager3() as example3, - ): - ... - -This new syntax uses the non LL(1) capacities of the new parser. -Check :pep:`617` for more details. - -(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou -in :issue:`12782` and :issue:`40334`.) - - -Better error messages in the parser ------------------------------------ - -When parsing code that contains unclosed parentheses or brackets the interpreter -now includes the location of the unclosed bracket of parentheses instead of displaying -*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. -For instance, consider the following code (notice the unclosed '{'): - -.. code-block:: python - - expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, - 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, - some_other_code = foo() - -previous versions of the interpreter reported confusing places as the location of -the syntax error: - -.. code-block:: text - - File "example.py", line 3 - some_other_code = foo() - ^ - SyntaxError: invalid syntax - -but in Python3.10 a more informative error is emitted: - -.. code-block:: text - - File "example.py", line 1 - expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, - ^ - SyntaxError: '{' was never closed - - -In a similar way, errors involving unclosed string literals (single and triple -quoted) now point to the start of the string instead of reporting EOF/EOL. - -These improvements are inspired by previous work in the PyPy interpreter. - -(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in -:issue:`40176`.) - - -PEP 634: Structural Pattern Matching ------------------------------------- - -Structural pattern matching has been added in the form of a *match statement* -and *case statements* of patterns with associated actions. Patterns -consist of sequences, mappings, primitive data types as well as class instances. -Pattern matching enables programs to extract information from complex data types, -branch on the structure of data, and apply specific actions based on different -forms of data. - -Syntax and operations -~~~~~~~~~~~~~~~~~~~~~ - -The generic syntax of pattern matching is:: - - match subject: - case : - - case : - - case : - - case _: - - -A match statement takes an expression and compares its value to successive -patterns given as one or more case blocks. Specifically, pattern matching -operates by: - - 1. using data with type and shape (the ``subject``) - 2. evaluating the ``subject`` in the ``match`` statement - 3. comparing the subject with each pattern in a ``case`` statement - from top to bottom until a match is confirmed. - 4. executing the action associated with the pattern of the confirmed - match - 5. If an exact match is not confirmed, the last case, a wildcard ``_``, - if provided, will be used as the matching case. If an exact match is - not confirmed and a wildcard case does not exists, the entire match - block is a no-op. - -Declarative approach -~~~~~~~~~~~~~~~~~~~~ - -Readers may be aware of pattern matching through the simple example of matching -a subject (data object) to a literal (pattern) with the switch statement found -in C, Java or JavaScript (and many other languages). Often the switch statement -is used for comparison of an object/expression with case statements containing -literals. - -More powerful examples of pattern matching can be found in languages, such as -Scala and Elixir. With structural pattern matching, the approach is "declarative" and -explicitly states the conditions (the patterns) for data to match. - -While an "imperative" series of instructions using nested "if" statements -could be used to accomplish something similar to structural pattern matching, -it is less clear than the "declarative" approach. Instead the "declarative" -approach states the conditions to meet for a match and is more readable through -its explicit patterns. While structural pattern matching can be used in its -simplest form comparing a variable to a literal in a case statement, its -true value for Python lies in its handling of the subject's type and shape. - -Simple pattern: match to a literal -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's look at this example as pattern matching in its simplest form: a value, -the subject, being matched to several literals, the patterns. In the example -below, ``status`` is the subject of the match statement. The patterns are -each of the case statements, where literals represent request status codes. -The associated action to the case is executed after a match:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - case _: - return "Something's wrong with the Internet" - -If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. -If the above function is passed a ``status`` of 500, the case statement with -``_`` will match as a wildcard, and "Something's wrong with the Internet" is -returned. -Note the last block: the variable name, ``_``, acts as a *wildcard* and insures -the subject will always match. The use of ``_`` is optional. - -You can combine several literals in a single pattern using ``|`` ("or"):: - - case 401 | 403 | 404: - return "Not allowed" - -Behavior without the wildcard -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If we modify the above example by removing the last case block, the example -becomes:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - -Without the use of ``_`` in a case statement, a match may not exist. If no -match exists, the behavior is a no-op. For example, if ``status`` of 500 is -passed, a no-op occurs. - -Patterns with a literal and variable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Patterns can look like unpacking assignments, and a pattern may be used to bind -variables. In this example, a data point can be unpacked to its x-coordinate -and y-coordinate:: - - # point is an (x, y) tuple - match point: - case (0, 0): - print("Origin") - case (0, y): - print(f"Y={y}") - case (x, 0): - print(f"X={x}") - case (x, y): - print(f"X={x}, Y={y}") - case _: - raise ValueError("Not a point") - -The first pattern has two literals, ``(0, 0)``, and may be thought of as an -extension of the literal pattern shown above. The next two patterns combine a -literal and a variable, and the variable *binds* a value from the subject -(``point``). The fourth pattern captures two values, which makes it -conceptually similar to the unpacking assignment ``(x, y) = point``. - -Patterns and classes -~~~~~~~~~~~~~~~~~~~~ - -If you are using classes to structure your data, you can use as a pattern -the class name followed by an argument list resembling a constructor. This -pattern has the ability to capture class attributes into variables:: - - class Point: - x: int - y: int - - def location(point): - match point: - case Point(x=0, y=0): - print("Origin is the point's location.") - case Point(x=0, y=y): - print(f"Y={y} and the point is on the y-axis.") - case Point(x=x, y=0): - print(f"X={x} and the point is on the x-axis.") - case Point(): - print("The point is located somewhere else on the plane.") - case _: - print("Not a point") - -Patterns with positional parameters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can use positional parameters with some builtin classes that provide an -ordering for their attributes (e.g. dataclasses). You can also define a specific -position for attributes in patterns by setting the ``__match_args__`` special -attribute in your classes. If it's set to ("x", "y"), the following patterns -are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: - - Point(1, var) - Point(1, y=var) - Point(x=1, y=var) - Point(y=var, x=1) - -Nested patterns -~~~~~~~~~~~~~~~ - -Patterns can be arbitrarily nested. For example, if our data is a short -list of points, it could be matched like this:: - - match points: - case []: - print("No points in the list.") - case [Point(0, 0)]: - print("The origin is the only point in the list.") - case [Point(x, y)]: - print(f"A single point {x}, {y} is in the list.") - case [Point(0, y1), Point(0, y2)]: - print(f"Two points on the Y axis at {y1}, {y2} are in the list.") - case _: - print("Something else is found in the list.") - -Complex patterns and the wildcard -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To this point, the examples have used ``_`` alone in the last case statement. -A wildcard can be used in more complex patterns, such as ``('error', code, _)``. -For example:: - - match test_variable: - case ('warning', code, 40): - print("A warning has been received.") - case ('error', code, _): - print(f"An error {code} occured.") - -In the above case, ``test_variable`` will match for ('error', code, 100) and -('error', code, 800). - -Guard -~~~~~ - -We can add an ``if`` clause to a pattern, known as a "guard". If the -guard is false, ``match`` goes on to try the next case block. Note -that value capture happens before the guard is evaluated:: - - match point: - case Point(x, y) if x == y: - print(f"The point is located on the diagonal Y=X at {x}.") - case Point(x, y): - print(f"Point is not on the diagonal.") - -Other Key Features -~~~~~~~~~~~~~~~~~~ - -Several other key features: - -- Like unpacking assignments, tuple and list patterns have exactly the - same meaning and actually match arbitrary sequences. Technically, - the subject must be an instance of ``collections.abc.Sequence``. - Therefore, an important exception is that patterns don't match iterators. - Also, to prevent a common mistake, sequence patterns don't match strings. - -- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, - *rest)`` work similar to wildcards in unpacking assignments. The - name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence - of at least two items without binding the remaining items. - -- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the - ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence - patterns, extra keys are ignored. A wildcard ``**rest`` is also - supported. (But ``**_`` would be redundant, so it not allowed.) - -- Subpatterns may be captured using the ``as`` keyword:: - - case (Point(x1, y1), Point(x2, y2) as p2): ... - - This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, - and p2 to the entire second item of the subject. - -- Most literals are compared by equality. However, the singletons ``True``, - ``False`` and ``None`` are compared by identity. - -- Named constants may be used in patterns. These named constants must be - dotted names to prevent the constant from being interpreted as a capture - variable:: - - from enum import Enum - class Color(Enum): - RED = 0 - GREEN = 1 - BLUE = 2 - - match color: - case Color.RED: - print("I see red!") - case Color.GREEN: - print("Grass is green") - case Color.BLUE: - print("I'm feeling the blues :(") - -For the full specification see :pep:`634`. Motivation and rationale -are in :pep:`635`, and a longer tutorial is in :pep:`636`. - - -New Features Related to Type Annotations -======================================== - -This section covers major changes affecting :pep:`484` type annotations and -the :mod:`typing` module. - - -PEP 563: Postponed Evaluation of Annotations Becomes Default ------------------------------------------------------------- - -In Python 3.7, postponed evaluation of annotations was added, -to be enabled with a ``from __future__ import annotations`` -directive. In 3.10 this became the default behavior, even -without that future directive. With this being default, all -annotations stored in :attr:`__annotations__` will be strings. -If needed, annotations can be resolved at runtime using -:func:`typing.get_type_hints`. See :pep:`563` for a full -description. Also, the :func:`inspect.signature` will try to -resolve types from now on, and when it fails it will fall back to -showing the string annotations. (Contributed by Batuhan Taskaya -in :issue:`38605`.) - - -PEP 604: New Type Union Operator --------------------------------- - -A new type union operator was introduced which enables the syntax ``X | Y``. -This provides a cleaner way of expressing 'either type X or type Y' instead of -using :data:`typing.Union`, especially in type hints (annotations). - -In previous versions of Python, to apply a type hint for functions accepting -arguments of multiple types, :data:`typing.Union` was used:: - - def square(number: Union[int, float]) -> Union[int, float]: - return number ** 2 - - -Type hints can now be written in a more succinct manner:: - - def square(number: int | float) -> int | float: - return number ** 2 - - -This new syntax is also accepted as the second argument to :func:`isinstance` -and :func:`issubclass`:: - - >>> isinstance(1, int | str) - True - -See :ref:`types-union` and :pep:`604` for more details. - -(Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) - - -PEP 612: Parameter Specification Variables ------------------------------------------- - -Two new options to improve the information provided to static type checkers for -:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. - -The first is the parameter specification variable. They are used to forward the -parameter types of one callable to another callable -- a pattern commonly -found in higher order functions and decorators. Examples of usage can be found -in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate -dependency of parameter types in such a precise manner. - -The second option is the new ``Concatenate`` operator. It's used in conjunction -with parameter specification variables to type annotate a higher order callable -which adds or removes parameters of another callable. Examples of usage can -be found in :class:`typing.Concatenate`. - -See :class:`typing.Callable`, :class:`typing.ParamSpec`, -:class:`typing.Concatenate` and :pep:`612` for more details. - -(Contributed by Ken Jin in :issue:`41559`.) - - -PEP 613: TypeAlias Annotation ------------------------------ - -:pep:`484` introduced the concept of type aliases, only requiring them to be -top-level unannotated assignments. This simplicity sometimes made it difficult -for type checkers to distinguish between type aliases and ordinary assignments, -especially when forward references or invalid types were involved. Compare:: - - StrCache = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant - -Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to -declare type aliases more explicitly:: - - StrCache: TypeAlias = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant - -See :pep:`613` for more details. - -(Contributed by Mikhail Golubev in :issue:`41923`.) - - -Other Language Changes -====================== - -* The :class:`int` type has a new method :meth:`int.bit_count`, returning the - number of ones in the binary expansion of a given integer, also known - as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) - -* The views returned by :meth:`dict.keys`, :meth:`dict.values` and - :meth:`dict.items` now all have a ``mapping`` attribute that gives a - :class:`types.MappingProxyType` object wrapping the original - dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) - -* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used - to require that all the iterables have an equal length. - -* Builtin and extension functions that take integer arguments no longer accept - :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other - objects that can be converted to integers only with a loss (e.g. that have - the :meth:`~object.__int__` method but do not have the - :meth:`~object.__index__` method). - (Contributed by Serhiy Storchaka in :issue:`37999`.) - -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will - correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. - (Contributed by Alex Shkop in :issue:`38302`.) - -* Assignment expressions can now be used unparenthesized within set literals - and set comprehensions, as well as in sequence indexes (but not slices). - -* Functions have a new ``__builtins__`` attribute which is used to look for - builtin symbols when a function is executed, instead of looking into - ``__globals__['__builtins__']``. The attribute is initialized from - ``__globals__["__builtins__"]`` if it exists, else from the current builtins. - (Contributed by Mark Shannon in :issue:`42990`.) - - -New Modules -=========== - -* None yet. - - -Improved Modules -================ - -argparse --------- - -Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. -(Contributed by Raymond Hettinger in :issue:`9694`.) - -base64 ------- - -Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the -Base32 Encoding with Extended Hex Alphabet. - -codecs ------- - -Add a :func:`codecs.unregister` function to unregister a codec search function. -(Contributed by Hai Shi in :issue:`41842`.) - -collections.abc ---------------- - -The ``__args__`` of the :ref:`parameterized generic ` for -:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. -:class:`collections.abc.Callable` generic now flattens type parameters, similar -to what :data:`typing.Callable` currently does. This means that -``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of -``(int, str, str)``; previously this was ``([int, str], str)``. To allow this -change, :class:`types.GenericAlias` can now be subclassed, and a subclass will -be returned when subscripting the :class:`collections.abc.Callable` type. Note -that a :exc:`TypeError` may be raised for invalid forms of parameterizing -:class:`collections.abc.Callable` which may have passed silently in Python 3.9. -(Contributed by Ken Jin in :issue:`42195`.) - -contextlib ----------- - -Add a :func:`contextlib.aclosing` context manager to safely close async generators -and objects representing asynchronously released resources. -(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) - -Add asynchronous context manager support to :func:`contextlib.nullcontext`. -(Contributed by Tom Gringauz in :issue:`41543`.) - -curses ------- - -The extended color functions added in ncurses 6.1 will be used transparently -by :func:`curses.color_content`, :func:`curses.init_color`, -:func:`curses.init_pair`, and :func:`curses.pair_content`. A new function, -:func:`curses.has_extended_color_support`, indicates whether extended color -support is provided by the underlying ncurses library. -(Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) - -The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if -they are provided by the underlying curses library. -(Contributed by Zackery Spytz in :issue:`39273`.) - -.. _distutils-deprecated: - -distutils ---------- - -The entire ``distutils`` package is deprecated, to be removed in Python -3.12. Its functionality for specifying package builds has already been -completely replaced by third-party packages ``setuptools`` and -``packaging``, and most other commonly used APIs are available elsewhere -in the standard library (such as :mod:`platform`, :mod:`shutil`, -:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate -any other functionality from ``distutils``, and applications that are -using other functions should plan to make private copies of the code. -Refer to :pep:`632` for discussion. - -The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. -The ``bdist_wheel`` command is now recommended to distribute binary packages -on Windows. -(Contributed by Victor Stinner in :issue:`42802`.) - -doctest -------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -encodings ---------- - -:func:`encodings.normalize_encoding` now ignores non-ASCII characters. -(Contributed by Hai Shi in :issue:`39337`.) - -gc --- - -Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and -:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) - -glob ----- - -Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and -:func:`~glob.iglob` which allow to specify the root directory for searching. -(Contributed by Serhiy Storchaka in :issue:`38144`.) - -importlib.metadata ------------------- - -Feature parity with ``importlib_metadata`` 3.7. - -:func:`importlib.metadata.entry_points` now provides a nicer experience -for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. - -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. - -inspect -------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -Added *globalns* and *localns* parameters in :func:`~inspect.signature` and -:meth:`inspect.Signature.from_callable` to retrieve the annotations in given -local and global namespaces. -(Contributed by Batuhan Taskaya in :issue:`41960`.) - -linecache ---------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -os --- - -Added :func:`os.cpu_count()` support for VxWorks RTOS. -(Contributed by Peixing Xin in :issue:`41440`.) - -Added a new function :func:`os.eventfd` and related helpers to wrap the -``eventfd2`` syscall on Linux. -(Contributed by Christian Heimes in :issue:`41001`.) - -Added :func:`os.splice()` that allows to move data between two file -descriptors without copying between kernel address space and user -address space, where one of the file descriptors must refer to a -pipe. (Contributed by Pablo Galindo in :issue:`41625`.) - -Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` -and :data:`~os.O_NOFOLLOW_ANY` for macOS. -(Contributed by Dong-hee Na in :issue:`43106`.) - -pathlib -------- - -Added slice support to :attr:`PurePath.parents `. -(Contributed by Joshua Cannon in :issue:`35498`) - -Added negative indexing support to :attr:`PurePath.parents -`. -(Contributed by Yaroslav Pankovych in :issue:`21041`) - -platform --------- - -Added :func:`platform.freedesktop_os_release()` to retrieve operation system -identification from `freedesktop.org os-release -`_ standard file. -(Contributed by Christian Heimes in :issue:`28468`) - -py_compile ----------- - -Added ``--quiet`` option to command-line interface of :mod:`py_compile`. -(Contributed by Gregory Schevchenko in :issue:`38731`.) - -pyclbr ------- - -Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` -objects in the tree returned by :func:`pyclbr.readline` and -:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. -(Contributed by Aviral Srivastava in :issue:`38307`.) - -shelve ------- - -The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default -instead of :mod:`pickle` protocol ``3`` when creating shelves. -(Contributed by Zackery Spytz in :issue:`34204`.) - -site ----- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -socket ------- - -The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. -(Contributed by Christian Heimes in :issue:`42413`.) - -sys ---- - -Add :data:`sys.orig_argv` attribute: the list of the original command line -arguments passed to the Python executable. -(Contributed by Victor Stinner in :issue:`23427`.) - -Add :data:`sys.stdlib_module_names`, containing the list of the standard library -module names. -(Contributed by Victor Stinner in :issue:`42955`.) - -_thread -------- - -:func:`_thread.interrupt_main` now takes an optional signal number to -simulate (the default is still :data:`signal.SIGINT`). -(Contributed by Antoine Pitrou in :issue:`43356`.) - -threading ---------- - -Added :func:`threading.gettrace` and :func:`threading.getprofile` to -retrieve the functions set by :func:`threading.settrace` and -:func:`threading.setprofile` respectively. -(Contributed by Mario Corchero in :issue:`42251`.) - -Add :data:`threading.__excepthook__` to allow retrieving the original value -of :func:`threading.excepthook` in case it is set to a broken or a different -value. -(Contributed by Mario Corchero in :issue:`42308`.) - -traceback ---------- - -The :func:`~traceback.format_exception`, -:func:`~traceback.format_exception_only`, and -:func:`~traceback.print_exception` functions can now take an exception object -as a positional-only argument. -(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) - -types ------ - -Reintroduced the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set -of types readily interpretable by type checkers. -(Contributed by Bas van Beek in :issue:`41810`.) - -typing ------- - -For major changes, see `New Features Related to Type Annotations`_. - -The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` -and to match the behavior of static type checkers specified in the PEP. - -1. ``Literal`` now de-duplicates parameters. -2. Equality comparisons between ``Literal`` objects are now order independent. -3. ``Literal`` comparisons now respects types. For example, - ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is - now ``False``. To support this change, the internally used type cache now - supports differentiating types. -4. ``Literal`` objects will now raise a :exc:`TypeError` exception during - equality comparisons if one of their parameters are not :term:`immutable`. - Note that declaring ``Literal`` with mutable parameters will not throw - an error:: - - >>> from typing import Literal - >>> Literal[{0}] - >>> Literal[{0}] == Literal[{False}] - Traceback (most recent call last): - File "", line 1, in - TypeError: unhashable type: 'set' - -(Contributed by Yurii Karabas in :issue:`42345`.) - -unittest --------- - -Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the -existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi -in :issue:`39385`.) - -urllib.parse ------------- - -Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as -query parameter separators in :func:`urllib.parse.parse_qs` and -:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with -newer W3C recommendations, this has been changed to allow only a single -separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected -functions internally. For more details, please see their respective -documentation. -(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) - -xml ---- - -Add a :class:`~xml.sax.handler.LexicalHandler` class to the -:mod:`xml.sax.handler` module. -(Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) - -zipimport ---------- -Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, -:meth:`zipimport.zipimporter.create_module`, and -:meth:`zipimport.zipimporter.exec_module`. -(Contributed by Brett Cannon in :issue:`42131`. - - -Optimizations -============= - -* Constructors :func:`str`, :func:`bytes` and :func:`bytearray` are now faster - (around 30--40% for small objects). - (Contributed by Serhiy Storchaka in :issue:`41334`.) - -* The :mod:`runpy` module now imports fewer modules. - The ``python3 -m module-name`` command startup time is 1.3x faster in - average. - (Contributed by Victor Stinner in :issue:`41006`.) - -* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It - is about 36% faster now for regular attributes and 44% faster for slots. - (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido - van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy - and MicroPython.) - -* When building Python with ``--enable-optimizations`` now - ``-fno-semantic-interposition`` is added to both the compile and link line. - This speeds builds of the Python interpreter created with ``--enable-shared`` - with ``gcc`` by up to 30%. See `this article - `_ - for more details. (Contributed by Victor Stinner and Pablo Galindo in - :issue:`38980`.) - - -* Function parameters and their annotations are no longer computed at runtime, - but rather at compilation time. They are stored as a tuple of strings at the - bytecode level. It is now around 2 times faster to create a function with - parameter annotations. (Contributed by Yurii Karabas and Inada Naoki - in :issue:`42202`) - -* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` - now sometimes use Crochemore & Perrin's "Two-Way" string searching - algorithm to avoid quadratic behavior on long strings. (Contributed - by Dennis Sweeney in :issue:`41972`) - -Deprecated -========== - -* Starting in this release, there will be a concerted effort to begin - cleaning up old import semantics that were kept for Python 2.7 - compatibility. Specifically, - :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), - :meth:`~importlib.abc.Loader.load_module` - (superseded by :meth:`~importlib.abc.Loader.exec_module`), - :meth:`~importlib.abc.Loader.module_repr` (which the import system - takes care of for you), the ``__package__`` attribute - (superseded by ``__spec__.parent``), the ``__loader__`` attribute - (superseded by ``__spec__.loader``), and the ``__cached__`` attribute - (superseded by ``__spec__.cached``) will slowly be removed (as well - as other classes and methods in :mod:`importlib`). - :exc:`ImportWarning` and/or :exc:`DeprecationWarning` will be raised - as appropriate to help identify code which needs updating during - this transition. - -* The entire ``distutils`` namespace is deprecated, to be removed in - Python 3.12. Refer to the :ref:`module changes ` - section for more information. - -* Non-integer arguments to :func:`random.randrange` are deprecated. - The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. - (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) - -* The various ``load_module()`` methods of :mod:`importlib` have been - documented as deprecated since Python 3.6, but will now also trigger - a :exc:`DeprecationWarning`. Use - :meth:`~importlib.abc.Loader.exec_module` instead. - (Contributed by Brett Cannon in :issue:`26131`.) - -* :meth:`zimport.zipimporter.load_module` has been deprecated in - preference for :meth:`~zipimport.zipimporter.exec_module`. - (Contributed by Brett Cannon in :issue:`26131`.) - -* The use of :meth:`~importlib.abc.Loader.load_module` by the import - system now triggers an :exc:`ImportWarning` as - :meth:`~importlib.abc.Loader.exec_module` is preferred. - (Contributed by Brett Cannon in :issue:`26131`.) - -* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python - 3.3, when it was made an alias to :class:`str`. It is now deprecated, - scheduled for removal in Python 3.12. - (Contributed by Erlend E. Aasland in :issue:`42264`.) - -* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now - deprecated, scheduled for removal in Python 3.12. Its use is strongly - discouraged by the SQLite3 documentation. See `the SQLite3 docs - `_ for more details. - If shared cache must be used, open the database in URI mode using the - ``cache=shared`` query parameter. - (Contributed by Erlend E. Aasland in :issue:`24464`.) - - -Removed -======= - -* Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, - ``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and - ``__rdivmod__`` of the :class:`complex` class. They always raised - a :exc:`TypeError`. - (Contributed by Serhiy Storchaka in :issue:`41974`.) - -* The ``ParserBase.error()`` method from the private and undocumented ``_markupbase`` - module has been removed. :class:`html.parser.HTMLParser` is the only subclass of - ``ParserBase`` and its ``error()`` implementation has already been removed in - Python 3.5. - (Contributed by Berker Peksag in :issue:`31844`.) - -* Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal - PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was - moved to the internal C API. - (Contributed by Victor Stinner in :issue:`42157`.) - -* Removed the ``parser`` module, which was deprecated in 3.9 due to the - switch to the new PEG parser, as well as all the C source and header files - that were only being used by the old parser, including ``node.h``, ``parser.h``, - ``graminit.h`` and ``grammar.h``. - -* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` - that were deprecated in 3.9 due to the switch to the new PEG parser. - -* Removed the ``formatter`` module, which was deprecated in Python 3.4. - It is somewhat obsolete, little used, and not tested. It was originally - scheduled to be removed in Python 3.6, but such removals were delayed until - after Python 2.7 EOL. Existing users should copy whatever classes they use - into their code. - (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) - -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. - (Contributed by Hai Shi in :issue:`42599`.) - -* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from - the :mod:`collections` module. - (Contributed by Victor Stinner in :issue:`37324`.) - -* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's - :doc:`high-level API <../library/asyncio-api-index>` following deprecation - in Python 3.8. The motivation behind this change is multifold: - - 1. This simplifies the high-level API. - 2. The functions in the high-level API have been implicitly getting the - current thread's running event loop since Python 3.7. There isn't a need to - pass the event loop to the API in most normal use cases. - 3. Event loop passing is error-prone especially when dealing with loops - running in different threads. - - Note that the low-level API will still accept ``loop``. - See `Changes in the Python API`_ for examples of how to replace existing code. - - (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley - in :issue:`42392`.) - - -Porting to Python 3.10 -====================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - - -Changes in the Python API -------------------------- - -* The *etype* parameters of the :func:`~traceback.format_exception`, - :func:`~traceback.format_exception_only`, and - :func:`~traceback.print_exception` functions in the :mod:`traceback` module - have been renamed to *exc*. - (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) - -* :mod:`atexit`: At Python exit, if a callback registered with - :func:`atexit.register` fails, its exception is now logged. Previously, only - some exceptions were logged, and the last exception was always silently - ignored. - (Contributed by Victor Stinner in :issue:`42639`.) - -* :class:`collections.abc.Callable` generic now flattens type parameters, similar - to what :data:`typing.Callable` currently does. This means that - ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of - ``(int, str, str)``; previously this was ``([int, str], str)``. Code which - accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account - for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms - of parameterizing :class:`collections.abc.Callable` which may have passed - silently in Python 3.9. - (Contributed by Ken Jin in :issue:`42195`.) - -* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` - instead of :exc:`DeprecationWarning` if the given parameter will not fit in - a 16-bit unsigned integer. - (Contributed by Erlend E. Aasland in :issue:`42393`.) - -* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's - :doc:`high-level API <../library/asyncio-api-index>` following deprecation - in Python 3.8. - - A coroutine that currently look like this:: - - async def foo(loop): - await asyncio.sleep(1, loop=loop) - - Should be replaced with this:: - - async def foo(): - await asyncio.sleep(1) - - If ``foo()`` was specifically designed *not* to run in the current thread's - running event loop (e.g. running in another thread's event loop), consider - using :func:`asyncio.run_coroutine_threadsafe` instead. - - (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley - in :issue:`42392`.) - -* The :data:`types.FunctionType` constructor now inherits the current builtins - if the *globals* dictionary has no ``"__builtins__"`` key, rather than using - ``{"None": None}`` as builtins: same behavior as :func:`eval` and - :func:`exec` functions. Defining a function with ``def function(...): ...`` - in Python is not affected, globals cannot be overriden with this syntax: it - also inherits the current builtins. - (Contributed by Victor Stinner in :issue:`42990`.) - -CPython bytecode changes -======================== - -* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations - instead of dictionary. - (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) - -Build Changes -============= - -* The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required - to build Python. - (Contributed by Victor Stinner in :issue:`36020`.) - -* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev - and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) - -* The :mod:`atexit` module must now always be built as a built-in module. - (Contributed by Victor Stinner in :issue:`42639`.) - -* Added ``--disable-test-modules`` option to the ``configure`` script: - don't build nor install test modules. - (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) - -* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If - specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` - wheel packages in this directory: if both are present, these wheel packages - are used instead of ensurepip bundled wheel packages. - - Some Linux distribution packaging policies recommend against bundling - dependencies. For example, Fedora installs wheel packages in the - ``/usr/share/python-wheels/`` directory and don't install the - ``ensurepip._bundled`` package. - - (Contributed by Victor Stinner in :issue:`42856`.) - -* Add a new configure ``--without-static-libpython`` option to not build the - ``libpythonMAJOR.MINOR.a`` static library and not install the ``python.o`` - object file. - - (Contributed by Victor Stinner in :issue:`43103`.) - -* The ``configure`` script now uses the ``pkg-config`` utility, if available, - to detect the location of Tcl/Tk headers and libraries. As before, those - locations can be explicitly specified with the ``--with-tcltk-includes`` - and ``--with-tcltk-libs`` configuration options. - (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) - - -C API Changes -============= - -New Features ------------- - -* The result of :c:func:`PyNumber_Index` now always has exact type :class:`int`. - Previously, the result could have been an instance of a subclass of ``int``. - (Contributed by Serhiy Storchaka in :issue:`40792`.) - -* Add a new :c:member:`~PyConfig.orig_argv` member to the :c:type:`PyConfig` - structure: the list of the original command line arguments passed to the - Python executable. - (Contributed by Victor Stinner in :issue:`23427`.) - -* The :c:func:`PyDateTime_DATE_GET_TZINFO` and - :c:func:`PyDateTime_TIME_GET_TZINFO` macros have been added for accessing - the ``tzinfo`` attributes of :class:`datetime.datetime` and - :class:`datetime.time` objects. - (Contributed by Zackery Spytz in :issue:`30155`.) - -* Add a :c:func:`PyCodec_Unregister` function to unregister a codec - search function. - (Contributed by Hai Shi in :issue:`41842`.) - -* The :c:func:`PyIter_Send` function was added to allow - sending value into iterator without raising ``StopIteration`` exception. - (Contributed by Vladimir Matveev in :issue:`41756`.) - -* Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. - (Contributed by Alex Gaynor in :issue:`41784`.) - -* Added :c:func:`PyModule_AddObjectRef` function: similar to - :c:func:`PyModule_AddObject` but don't steal a reference to the value on - success. - (Contributed by Victor Stinner in :issue:`1635741`.) - -* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the - reference count of an object and return the object. - (Contributed by Victor Stinner in :issue:`42262`.) - -* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` - functions now accept a single class as the *bases* argument. - (Contributed by Serhiy Storchaka in :issue:`42423`.) - -* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` - slot. - (Contributed by Hai Shi in :issue:`41832`.) - -* The :c:func:`PyType_GetSlot` function can accept static types. - (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) - -* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an - object is an instance of :class:`set` but not an instance of a subtype. - (Contributed by Pablo Galindo in :issue:`43277`.) - -* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number - to simulate. - (Contributed by Antoine Pitrou in :issue:`43356`.) - - -Porting to Python 3.10 ----------------------- - -* The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use - :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use - ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. - See :ref:`Parsing arguments and building values - ` and the :pep:`353`. - (Contributed by Victor Stinner in :issue:`40943`.) - -* Since :c:func:`Py_REFCNT()` is changed to the inline static function, - ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: - see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward - compatibility, this macro can be used:: - - #if PY_VERSION_HEX < 0x030900A4 - # define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) - #endif - - (Contributed by Victor Stinner in :issue:`39573`.) - -* Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed - for historical reason. It is no longer allowed. - (Contributed by Victor Stinner in :issue:`40839`.) - -* ``PyUnicode_FromUnicode(NULL, size)`` and ``PyUnicode_FromStringAndSize(NULL, size)`` - raise ``DeprecationWarning`` now. Use :c:func:`PyUnicode_New` to allocate - Unicode object without initial data. - (Contributed by Inada Naoki in :issue:`36346`.) - -* The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API - ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. - (Contributed by Victor Stinner in :issue:`42157`.) - -* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and - :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before - :c:func:`Py_Initialize` (before Python is initialized). Use the new - :ref:`Python Initialization Configuration API ` to get the - :ref:`Python Path Configuration. `. - (Contributed by Victor Stinner in :issue:`42260`.) - -* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. - For example, ``x = PyList_SET_ITEM(a, b, c)`` and - ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents - bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. - (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) - -* The non-limited API files ``odictobject.h``, ``parser_interface.h``, - ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, - ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` - directory. These files must not be included directly, as they are already - included in ``Python.h``: :ref:`Include Files `. If they have - been included directly, consider including ``Python.h`` instead. - (Contributed by Nicholas Sim in :issue:`35134`) - -Deprecated ----------- - -* The ``PyUnicode_InternImmortal()`` function is now deprecated - and will be removed in Python 3.12: use :c:func:`PyUnicode_InternInPlace` - instead. - (Contributed by Victor Stinner in :issue:`41692`.) - -Removed -------- - -* ``PyObject_AsCharBuffer()``, ``PyObject_AsReadBuffer()``, ``PyObject_CheckReadBuffer()``, - and ``PyObject_AsWriteBuffer()`` are removed. Please migrate to new buffer protocol; - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release`. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. - (Contributed by Inada Naoki in :issue:`41123`.) - - * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or - :c:macro:`PyUnicode_GET_LENGTH` - * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or - :c:func:`PyUnicode_FromFormat` - * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use - :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` - * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` - * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` - * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use - :c:func:`PyUnicode_FindChar` - -* Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``PyLong_FromUnicode()``. Please migrate to :c:func:`PyLong_FromUnicodeObject`. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``PyUnicode_AsUnicodeCopy()``. Please use :c:func:`PyUnicode_AsUCS4Copy` or - :c:func:`PyUnicode_AsWideCharString` - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``_Py_CheckRecursionLimit`` variable: it has been replaced by - ``ceval.recursion_limit`` of the :c:type:`PyInterpreterState` structure. - (Contributed by Victor Stinner in :issue:`41834`.) - -* Removed undocumented macros ``Py_ALLOW_RECURSION`` and - ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the - :c:type:`PyInterpreterState` structure. - (Contributed by Serhiy Storchaka in :issue:`41936`.) - -* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing - Python already implicitly installs signal handlers: see - :c:member:`PyConfig.install_signal_handlers`. - (Contributed by Victor Stinner in :issue:`41713`.) From 7b65fbce94c3898d17d0f2f9d5f41e8421204b1c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 15:22:46 +0000 Subject: [PATCH 09/24] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst diff --git a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst new file mode 100644 index 000000000000000..eb89caa3976db45 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst @@ -0,0 +1,2 @@ +Add an Barrier object in synchronization primitives of asyncio Lib +in order to be consistant with Barrier from *threading* and *multiprocessing* libs \ No newline at end of file From 9b60d076853303b5bd54ff88bc26016919a0183a Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 2 Apr 2021 14:57:51 +0200 Subject: [PATCH 10/24] Update 2021-03-31-15-22-45.bpo-43352.nSjMuE.rst --- .../next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst index eb89caa3976db45..e53ba28b64099fd 100644 --- a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst +++ b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst @@ -1,2 +1 @@ -Add an Barrier object in synchronization primitives of asyncio Lib -in order to be consistant with Barrier from *threading* and *multiprocessing* libs \ No newline at end of file +Add an Barrier object in synchronization primitives of *asyncio* Lib in order to be consistant with Barrier from *threading* and *multiprocessing* libs* From 576cd53c206f05b586611c6db562bdbd4c71af56 Mon Sep 17 00:00:00 2001 From: Duprat Date: Sun, 11 Apr 2021 15:36:40 +0200 Subject: [PATCH 11/24] Update documentation about add of Barrier object Add Barrier object as a clone of threading.Barrier --- Doc/library/asyncio-api-index.rst | 3 +++ Doc/library/asyncio-sync.rst | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index f558724d4a3ff65..3465a2971209200 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -186,6 +186,9 @@ Threading-like synchronization primitives that can be used in Tasks. * - :class:`BoundedSemaphore` - A bounded semaphore. + * - :class:`Barrier` + - A barrier object. + .. rubric:: Examples diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index d12630afc6a3261..ac6e84d070850eb 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -28,6 +28,7 @@ asyncio has the following basic synchronization primitives: * :class:`Condition` * :class:`Semaphore` * :class:`BoundedSemaphore` +* :class:`Barrier` --------- @@ -325,6 +326,22 @@ BoundedSemaphore a :exc:`ValueError` in :meth:`~Semaphore.release` if it increases the internal counter above the initial *value*. + +Barrier +======= + +.. class:: Barrier(parties, action=None) + + A barrier object. Not thread-safe. + + Create a barrier object for *parties* number of tasks. An *action*, when + provided, is a callable to be called by one of the tasks when they are + released. + + A barrier object: a clone of :class:`threading.Barrier`. + + .. versionadded:: 3.10 + --------- From 6106f21de1685022970a77ab0440e9da4d41fea3 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 30 Apr 2021 14:47:28 +0200 Subject: [PATCH 12/24] Update asyncio-sync.rst Describe when action is called --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index ac6e84d070850eb..f7ff876cf61dd06 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -335,7 +335,7 @@ Barrier A barrier object. Not thread-safe. Create a barrier object for *parties* number of tasks. An *action*, when - provided, is a callable to be called by one of the tasks when they are + provided, is a callable to be called by the last waiting task when they are released. A barrier object: a clone of :class:`threading.Barrier`. From 03845614e2bac94aa9ab746751a813aa6cd8f52d Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 13:54:00 +0200 Subject: [PATCH 13/24] Update locks.py first feedbacks from Emmanuel Arias --- Lib/asyncio/locks.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e7f2561468e8cf1..57d998768147ca7 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,6 +1,7 @@ """Synchronization primitives.""" -__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', + 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections @@ -431,9 +432,10 @@ def release(self): # since the previous cycle. In addition, a 'resetting' state exists which is # similar to 'draining' except that tasks leave with a BrokenBarrierError, # and a 'broken' state in which all tasks get the exception. - +# +# Asyncio equivalent to threading.Barrier class Barrier(mixins._LoopBoundMixin): - """Asynchronous equivalent to threading.Barrier + """Asyncio equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization @@ -495,10 +497,9 @@ async def _block (self): # It is draining or resetting, wait until done await self._blocking.wait() - #see if the barrier is in a broken state + # see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError - assert self._state == 0, repr(self) # Optionally run the 'action' and release the tasks waiting # in the barrier. @@ -512,9 +513,7 @@ def _release(self): self._waiting.set() except: #an exception during the _action handler. Break and reraise - self._state = -2 - self._blocking.clear() - self._waiting.set() + self.abort() raise # Wait in the barrier until we are released. Raise an exception @@ -524,7 +523,6 @@ async def _wait(self): # no timeout so if self._state < 0: # resetting or broken raise BrokenBarrierError - assert self._state == 1, repr(self) # If we are the last tasks to exit the barrier, signal any tasks # waiting for the barrier to drain. From 16515ef7caee0934829db8e54cd988929016e172 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 13:56:05 +0200 Subject: [PATCH 14/24] Update test_locks.py Remove "import functools" --- Lib/test/test_asyncio/test_locks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index ec90698b6155aad..85f0a4d4d4428aa 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -6,7 +6,6 @@ import asyncio from test.test_asyncio import utils as test_utils -import functools STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' From fa217fdb83bef6c11265ba901bd9ec63b8e5384c Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 16:55:53 +0200 Subject: [PATCH 15/24] Update asyncio-sync.rst Suggestion of Emmanuel Arias: Add sample and description about Barrier Class --- Doc/library/asyncio-sync.rst | 124 +++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index f7ff876cf61dd06..0eab76810948b82 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -330,17 +330,129 @@ BoundedSemaphore Barrier ======= -.. class:: Barrier(parties, action=None) + .. versionadded:: 3.10 A barrier object. Not thread-safe. - Create a barrier object for *parties* number of tasks. An *action*, when - provided, is a callable to be called by the last waiting task when they are - released. + This class, a clone of :class:`threading.Barrier`, provides a simple synchronization + primitive for use by a fixed number of tasks that need to wait for each other. + Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method + and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. + At this point, the tasks are released simultaneously. - A barrier object: a clone of :class:`threading.Barrier`. + The barrier can be reused any number of times for the same number of tasks. - .. versionadded:: 3.10 + .. _asyncio_example_barrier: + + Example:: + + + import asyncio + import os + import random + + DELAYS = (1.0, 4.0, 2.0, 0.5) + + def job_done(): + msg = "ACTION: All files are removed" + print(f'{os.linesep}{msg:-^80s}{os.linesep}') + + async def remove_file(barrier, filename): + print(f'Remove "{filename}" ...') + + # remove filename + await asyncio.sleep(random.choice(DELAYS)) # simulate operation + print(f'\t"{filename}" is removed') + + return await barrier.wait() + + async def remove_folder(barrier, folder): + print(f'Waiting for remove all files from folder "{folder}" ...') + + # wait for all removed files + await barrier.wait() + + # remove folder + await asyncio.sleep(1.0) # simulates operation + print(f'Folder "{folder}" is now removed ...') + + return -1 + + async def main(folder, files, nb_files): + # Create a Barrier object with parties equal to ``nb_files+1`` + # corresponding to the number of all removing items, + # and an action as a print message. + barrier = asyncio.Barrier(nb_files+1, action=job_done) + + # Add remove_file task for each filename. + tasks = [asyncio.create_task(remove_file(barrier, f)) for f in files] + + # Add remove_folder task for the folder. + tasks.append(asyncio.create_task(remove_folder(barrier, folder))) + + # Wait until folder+filenames are removed. + await asyncio.gather(*tasks) + + asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) + + +.. class:: Barrier(parties, action=None) + + Create a barrier object for *parties* number of tasks. A callable *action*, when + provided, is called once at the begining of draining. + + .. coroutinemethod:: wait() + + Pass the barrier. When all the tasks party to the barrier have called + this function, they are all released simultaneously. + + The return value is an integer in the range 0 to *parties* -- 1, different + for each task. This can be used to select a task to do some special + housekeeping, e.g.:: + + ... + file_position = await barrier.wait() + if file_position == 0: + # Only one tasks needs to print this + print(f"I have the position #{file_position}") + + If an *action* was provided to the constructor, the last calling task of :meth:`wait` method will + have called it prior to being released. Should this call raise an error, + the barrier is put into the broken state. + + This method may raise a :class:`BrokenBarrierError` exception if the + barrier is broken or reset while a task is waiting. + + .. method:: reset() + + Return the barrier to the default, empty state. Any tasks waiting on it + will receive the :class:`BrokenBarrierError` exception. + + If a barrier is broken it may be better to just leave it and create a new one. + + .. method:: abort() + + Put the barrier into a broken state. This causes any active or future + calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + Use this for example if one of the taks needs to abort, to avoid infinite waiting tasks. + + .. attribute:: parties + + The number of tasks required to pass the barrier. + + .. attribute:: n_waiting + + The number of tasks currently waiting in the barrier. + + .. attribute:: broken + + A boolean that is ``True`` if the barrier is in the broken state. + + +.. exception:: BrokenBarrierError + + This exception, a subclass of :exc:`RuntimeError`, is raised when the + :class:`Barrier` object is reset or broken. --------- From a97f4a34a71d34c717879a6b0aa477dbb9d70767 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 21:28:14 +0200 Subject: [PATCH 16/24] Update asyncio-sync.rst bpo-43352 - Add a sample and description for Barrier Object Minor updates as remove crlf --- Doc/library/asyncio-sync.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 0eab76810948b82..263d43d6cd854ff 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -346,7 +346,6 @@ Barrier Example:: - import asyncio import os import random @@ -395,7 +394,6 @@ Barrier asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) - .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A callable *action*, when From b598470f186e72be2b4772df442e0d879abc247a Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 18 May 2021 08:35:21 +0200 Subject: [PATCH 17/24] Update asyncio-sync.rst bpo-43352 Suppress ` in a comment of the example --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 263d43d6cd854ff..2cc437c0abfe7c1 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -378,7 +378,7 @@ Barrier return -1 async def main(folder, files, nb_files): - # Create a Barrier object with parties equal to ``nb_files+1`` + # Create a Barrier object with parties equal to nb_files+1 # corresponding to the number of all removing items, # and an action as a print message. barrier = asyncio.Barrier(nb_files+1, action=job_done) From 309810587488696f2d0b1f39e300586699b01a28 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 15:55:25 +0200 Subject: [PATCH 18/24] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a class BarrierTests to test Barrier object Run i17 tests ncluding one about Barrier does not accept loop parameter --- Lib/test/test_asyncio/test_locks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 85f0a4d4d4428aa..dbe36d6da5362e2 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,7 +11,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # new line dedicated to repr barrier + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -958,6 +958,14 @@ def cancel_coros(self): task.cancel() test_utils.run_briefly(self.loop) + def test_barrier_doesnt_accept_loop_parameter(self): + with self.assertRaisesRegex( + TypeError, + rf'As of 3.10, the \*loop\* parameter was removed from ' + rf'Barrier\(\) since it is no longer necessary' + ): + asyncio.Barrier(1, loop=self.loop) + def test_repr(self): async def coro(): try: @@ -967,7 +975,7 @@ async def coro(): else: return await asyncio.sleep(1, True) - self.N = 1011001 + self.N = 999 barrier = asyncio.Barrier(self.N) self.assertTrue("[unset," in repr(barrier)) self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) From 2f55acc14525dcc3fe78ef0f94427cd5e05e98d8 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:05:55 +0200 Subject: [PATCH 19/24] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add the Barrier object in Synchronization section Add a link to a Barrier sample in Examples sub-section Add the BrokenBarrierError Exception in Exception section --- Doc/library/asyncio-api-index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index 3465a2971209200..33c2b119dfb84a6 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -194,6 +194,8 @@ Threading-like synchronization primitives that can be used in Tasks. * :ref:`Using asyncio.Event `. +* :ref:`Using asyncio.Barrier `. + * See also the documentation of asyncio :ref:`synchronization primitives `. @@ -214,6 +216,9 @@ Exceptions * - :exc:`asyncio.CancelledError` - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + * - :exc:`asyncio.BrokenBarrierError` + - Raised when a Barrier is broken. See also :meth:`Barrier.wait`. + .. rubric:: Examples From f056865137ef85b399bbd481a37648c082f58902 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:09:01 +0200 Subject: [PATCH 20/24] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a complete documentation inspired from threading.Barrier Add a sample to show how use a Barrier Object --- Doc/library/asyncio-sync.rst | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 2cc437c0abfe7c1..63cfcd47f343161 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -334,7 +334,7 @@ Barrier A barrier object. Not thread-safe. - This class, a clone of :class:`threading.Barrier`, provides a simple synchronization + This class, as a clone of :class:`threading.Barrier`, provides a simple synchronization primitive for use by a fixed number of tasks that need to wait for each other. Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. @@ -351,29 +351,31 @@ Barrier import random DELAYS = (1.0, 4.0, 2.0, 0.5) + NB = 10 def job_done(): - msg = "ACTION: All files are removed" + msg = 'ACTION: All files removed' print(f'{os.linesep}{msg:-^80s}{os.linesep}') async def remove_file(barrier, filename): - print(f'Remove "{filename}" ...') + print(f'Remove {filename!r} ...') - # remove filename - await asyncio.sleep(random.choice(DELAYS)) # simulate operation - print(f'\t"{filename}" is removed') + # remove filename (simulation) + await asyncio.sleep(random.choice(DELAYS)) + print(f'\tFile {filename!r} is removed') + # wait for others files return await barrier.wait() async def remove_folder(barrier, folder): - print(f'Waiting for remove all files from folder "{folder}" ...') + print(f'Waiting for remove all files from folder {folder!r} ...') # wait for all removed files await barrier.wait() - # remove folder - await asyncio.sleep(1.0) # simulates operation - print(f'Folder "{folder}" is now removed ...') + # remove folder (simulation) + await asyncio.sleep(1.0) + print(f'Folder {folder!r} is now removed ...') return -1 @@ -392,12 +394,12 @@ Barrier # Wait until folder+filenames are removed. await asyncio.gather(*tasks) - asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) + asyncio.run(main("Datas", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A callable *action*, when - provided, is called once at the begining of draining. + provided, is called once just before the release phasis (draining state). .. coroutinemethod:: wait() @@ -414,9 +416,9 @@ Barrier # Only one tasks needs to print this print(f"I have the position #{file_position}") - If an *action* was provided to the constructor, the last calling task of :meth:`wait` method will - have called it prior to being released. Should this call raise an error, - the barrier is put into the broken state. + If an *action* was provided to the constructor, the last calling task of + :meth:`wait` method will have called it prior to being released. + Should this call raise an error, the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the barrier is broken or reset while a task is waiting. @@ -432,7 +434,8 @@ Barrier Put the barrier into a broken state. This causes any active or future calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. - Use this for example if one of the taks needs to abort, to avoid infinite waiting tasks. + Use this for example if one of the taks needs to abort, to avoid infinite + waiting tasks. .. attribute:: parties From d9008f95f9a7198c35ca95112bdf8897b3ff3f04 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:14:58 +0200 Subject: [PATCH 21/24] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a Barrier object as a clone from thraeding Barrier Update of __all__ variable to include this new class and the related BrokenBarrierError exception --- Lib/asyncio/locks.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 57d998768147ca7..eec4dafed5a903e 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -445,33 +445,37 @@ class Barrier(mixins._LoopBoundMixin): def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. - 'action' is a callable which, when supplied, will be called by one of - the tasks after they have all entered the barrier and just prior to - releasing them all. + 'action' is a callable which, when supplied, will be called by + the last task calling the wait() method, + just prior to releasing them all. """ super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block new tasks while waiting tasks are draining or broken + self._blocking = Event() # used block new tasks while waiting + # tasks are draining or broken self._action = action self._parties = parties - self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._state = 0 # 0 filling, 1, draining, + # -1 resetting, -2 broken self._count = 0 # count waiting tasks def __repr__(self): res = super().__repr__() _wait = 'set' if self._waiting.is_set() else 'unset' _block = 'set' if self._blocking.is_set() else 'unset' - extra = f'{_wait}, count:{self._count}/{self._parties}, {_block}, state:{self._state}' + extra = f'{_wait}, count:{self._count}/{self._parties}' + extra += f', {_block}, state:{self._state}' return f'<{res[1:-1]} [{extra}]>' async def wait(self): """Wait for the barrier. When the specified number of tasks have started waiting, they are all - simultaneously awoken. If an 'action' was provided for the barrier, one - of the tasks will have executed that callback prior to returning. + simultaneously awoken. If an 'action' was provided for the barrier, the + last task calling this method will have executed that callback prior to + returning. Returns an individual index number from 0 to 'parties-1'. """ await self._block() # Block while the barrier drains or resets. @@ -520,6 +524,7 @@ def _release(self): # if the barrier is reset or broken. async def _wait(self): await self._waiting.wait() + # no timeout so if self._state < 0: # resetting or broken raise BrokenBarrierError From 71b923ca099cf7b4635348a284bc1ea355936227 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 21:42:40 +0200 Subject: [PATCH 22/24] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a class BarrierTests to test Barrier object Run 17 tests including one about Barrier does not accept loop parameter --- Lib/test/test_asyncio/test_locks.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index dbe36d6da5362e2..b0d93ad55b77038 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -69,16 +69,6 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) - # Barrier object has a positional paramater - # so tests alone - cls = asyncio.Barrier - with self.assertRaisesRegex( - TypeError, - rf'As of 3.10, the \*loop\* parameter was removed from ' - rf'{cls.__name__}\(\) since it is no longer necessary' - ): - cls(1, loop=self.loop) - def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) From e5bb0f11c3ea2c0e1878eb0f4cf0730dfdfacb58 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 19 Apr 2022 17:58:54 +0200 Subject: [PATCH 23/24] Update test_locks.py --- Lib/test/test_asyncio/test_locks.py | 407 ---------------------------- 1 file changed, 407 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 7046795940e24f4..415cbe570fde98d 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1517,412 +1517,5 @@ async def coro(): self.assertEqual(barrier1.n_waiting, 0) -class BarrierTests(test_utils.TestCase): - """ - Tests for Barrier objects. - """ - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() - self.N = 5 - - async def run_coros(self, n, coro): - tasks = [self.loop.create_task(coro()) for _ in range(n)] - res = await asyncio.gather(*tasks) - return res, tasks - - def cancel_coros(self): - for task in asyncio.all_tasks(self.loop): - task.cancel() - test_utils.run_briefly(self.loop) - - def test_barrier_doesnt_accept_loop_parameter(self): - with self.assertRaisesRegex( - TypeError, - rf'As of 3.10, the \*loop\* parameter was removed from ' - rf'Barrier\(\) since it is no longer necessary' - ): - asyncio.Barrier(1, loop=self.loop) - - def test_repr(self): - async def coro(): - try: - i = await barrier.wait() - except: - ... - else: - return await asyncio.sleep(1, True) - - self.N = 999 - barrier = asyncio.Barrier(self.N) - self.assertTrue("[unset," in repr(barrier)) - self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(repr(barrier).endswith('unset, state:0]>')) - - nb_waits = 3 - tasks = [] - for _ in range(nb_waits): - tasks.append(self.loop.create_task(coro())) - test_utils.run_briefly(self.loop) - self.assertTrue("[unset," in repr(barrier)) - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith('unset, state:0]>')) - - barrier.reset() - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue("[set," in repr(barrier)) - self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith('unset, state:-1]>')) - test_utils.run_briefly(self.loop) - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue("[unset," in repr(barrier)) - self.assertTrue(f"count:0/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith('set, state:0]>')) - test_utils.run_briefly(self.loop) - - barrier.abort() - test_utils.run_briefly(self.loop) - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"0/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith(':-2]>')) - - self.cancel_coros() - - def test_init(self): - self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) - self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - - def test_wait_task_once(self): - self.N = 1 - barrier = asyncio.Barrier(self.N) - self.assertEqual(barrier.n_waiting, 0) - r1 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - r2 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - self.assertEqual(r1, r2) - self.assertFalse(barrier.broken) - - def test_wait_task_by_task(self): - self.N = 3 - barrier = asyncio.Barrier(self.N) - self.assertEqual(barrier.n_waiting, 0) - self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 1) - self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 2) - self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_wait_tasks_twice(self): - results = [] - async def coro(): - await barrier.wait() - results.append(True) - await barrier.wait() - results.append(False) - - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertEqual(len(results), self.N*2) - self.assertEqual(results.count(True), self.N) - self.assertEqual(results.count(False), self.N) - self.assertFalse(barrier.broken) - self.assertEqual(barrier.n_waiting, 0) - - def test_wait_return_from_wait(self): - results1 = [] - results2 = [] - async def coro(): - await barrier.wait() - results1.append(True) - i = await barrier.wait() - results2.append(True) - return i - - barrier = asyncio.Barrier(self.N) - res = self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertEqual(len(results1), self.N) - self.assertTrue(all(results1)) - self.assertEqual(len(results2), self.N) - self.assertTrue(all(results2)) - self.assertEqual(sum(res[0]), sum(range(self.N))) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - - def test_wait_repeat_more(self, more=5): - async def coro(result, value): - ret = await barrier.wait() - result.append(value) - - results = [[] for _ in range(more)] - barrier = asyncio.Barrier(self.N) - for i in range(more): - _ = [self.loop.create_task(coro(results[i], value)) for value in range(self.N)] - test_utils.run_briefly(self.loop) - - self.assertEqual(len(results[i]), self.N) - self.assertEqual(sum(results[i]), sum(range(self.N))) - if i > 0: - self.assertEqual(sum(results[i]), sum(results[i-1])) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_wait_state_draining(self): - result = [] - state = [] - async def coro(): - i = await barrier.wait() - if i == 0: - result.append(True) - # Do we need to add a Barrie.draining() property ? - state.append(repr(barrier).endswith("state:1]>")) - - rest = self.N-1 - nb_tasks = self.N + rest - barrier = asyncio.Barrier(self.N) - for _ in range(nb_tasks): - self.loop.create_task(coro()) - for _ in range((nb_tasks//self.N)+1): - test_utils.run_briefly(self.loop) - self.assertEqual(len(result), nb_tasks//self.N) - self.assertTrue(all(result)) - self.assertTrue(all(state)) - self.assertEqual(barrier.n_waiting, rest) - self.assertFalse(barrier.broken) - - self.cancel_coros() - - def test_wait_tasks_cancel_one_task(self): - results = [] - async def coro(): - await barrier.wait() - results.append(True) - - self.N=3 - barrier = asyncio.Barrier(self.N) - t1 = self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 1) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 2) - - t1.cancel() - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 1) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 2) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.assertEqual(len(results), self.N) - self.assertTrue(all(results)) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_wait_action_callback(self): - result = [] - async def coro(): - ret = await barrier.wait() - result.append(True) - - result1 = [] - self.N = 2 - barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - - self.assertEqual(len(result1), 1) - self.assertTrue(result1[0]) - self.assertEqual(len(result), 2) - self.assertTrue(all(result)) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_wait_action_callback_more(self): - result = [] - async def coro(): - ret = await barrier.wait() - result.append(True) - - result1 = [] - - self.N = 2 - more = 3 - barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - for _ in range(more): - _ = [self.loop.create_task(coro()) for _ in range(self.N)] - test_utils.run_briefly(self.loop) - - self.assertEqual(len(result1), more) - self.assertTrue(all(result1)) - self.assertEqual(len(result), self.N*more) - self.assertTrue(all(result)) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_wait_action_callback_error_broken(self): - ERROR = ZeroDivisionError - results1 = [] - results2 = [] - async def coro(): - try: - ret = await barrier.wait() - except ERROR: - results1.append(False) - except: - results2.append(True) - - def raise_except(): - raise ERROR - - barrier = asyncio.Barrier(self.N, lambda: raise_except()) - _ = [self.loop.create_task(coro()) for _ in range(self.N)] - test_utils.run_briefly(self.loop) - self.assertEqual(len(results1), 1) - self.assertFalse(results1[0]) - self.assertEqual(len(results2), self.N-1) - self.assertTrue(all(results2)) - self.assertEqual(barrier.n_waiting, 0) - - self.assertTrue(barrier.broken) - - def test_reset(self): - results1 = [] - results2 = [] - async def coro(): - try: - await barrier.wait() - except asyncio.BrokenBarrierError: - results1.append(True) - finally: - await barrier.wait() - results2.append(True) - - async def coro2(): - await barrier.wait() - results2.append(True) - - - barrier = asyncio.Barrier(self.N) - _ = [self.loop.create_task(coro()) for _ in range(self.N-1)] - test_utils.run_briefly(self.loop) - barrier.reset() - test_utils.run_briefly(self.loop) - self.loop.create_task(coro2()) - test_utils.run_briefly(self.loop) - self.assertFalse(barrier.broken) - self.assertEqual(len(results1), self.N-1) - self.assertEqual(len(results2), self.N) - self.assertEqual(barrier.n_waiting, 0) - - self.assertFalse(barrier.broken) - - def test_reset_and_wait(self): - results1 = [] - results2 = [] - results3 = [] - count = 0 - async def coro(): - nonlocal count - - i = await barrier1.wait() - count += 1 - if count == self.N: # i == self.N//2: - barrier.reset() - else: - try: - await barrier.wait() - results1.append(True) - except Exception as e: - results2.append(True) - - # Now, pass the barrier again - await barrier1.wait() - results3.append(True) - - barrier = asyncio.Barrier(self.N) - barrier1 = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertFalse(barrier.broken) - self.assertEqual(len(results1), 0) - self.assertEqual(len(results2), self.N-1) - self.assertEqual(len(results3), self.N) - self.assertEqual(barrier.n_waiting, 0) - - self.assertFalse(barrier.broken) - - def test_abort_broken(self): - results1 = [] - results2 = [] - async def coro(): - try: - i = await barrier.wait() - if i == self.N//2: - raise RuntimeError - await barrier.wait() - results1.append(True) - except asyncio.BrokenBarrierError: - results2.append(True) - except RuntimeError: - barrier.abort() - - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertTrue(barrier.broken) - self.assertEqual(len(results1), 0) - self.assertEqual(len(results2), self.N-1) - self.assertEqual(barrier.n_waiting, 0) - - self.assertTrue(barrier.broken) - - def test_abort_and_reset(self): - results1 = [] - results2 = [] - results3 = [] - async def coro(): - try: - i = await barrier.wait() - if i == self.N//2: - raise RuntimeError - await barrier.wait() - results1.append(True) - except asyncio.BrokenBarrierError: - results2.append(True) - except RuntimeError: - barrier.abort() - - # Synchronize and reset the barrier. Must synchronize first so - # that everyone has left it when we reset, and after so that no - # one enters it before the reset. - i = await barrier2.wait() - if i == self.N//2: - barrier.reset() - await barrier2.wait() - await barrier.wait() - results3.append(True) - - barrier = asyncio.Barrier(self.N) - barrier2 = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertFalse(barrier.broken) - self.assertEqual(len(results1), 0) - self.assertEqual(len(results2), self.N-1) - self.assertEqual(len(results3), self.N) - self.assertEqual(barrier.n_waiting, 0) - - self.assertFalse(barrier.broken) - - if __name__ == '__main__': unittest.main() From 52d5cafefbaef89e1790ee41dee5ea47867861d5 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 27 Nov 2023 16:24:28 +0100 Subject: [PATCH 24/24] initial commit --- Lib/asyncio/queues.py | 55 ++++- Lib/test/test_asyncio/test_queues.py | 354 ++++++++++++++++++++++++++- 2 files changed, 399 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df561ba8..db5eb4973ec862b 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,5 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty', + 'QueueFullWithPendingPutTasks', 'QueueEmptyWithPendingGetTasks') import collections import heapq @@ -12,11 +13,17 @@ class QueueEmpty(Exception): """Raised when Queue.get_nowait() is called on an empty Queue.""" pass +class QueueEmptyWithPendingGetTasks(QueueEmpty): + """Raised when the Queue.get_nowait() method is called on not empty Queue._getters""" + pass class QueueFull(Exception): """Raised when the Queue.put_nowait() method is called on a full Queue.""" pass +class QueueFullWithPendingPutTasks(QueueFull): + """Raised when the Queue.put_nowait() method is called on not empty Queue._putters""" + pass class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -57,8 +64,9 @@ def _put(self, item): def _wakeup_next(self, waiters): # Wake up the next waiter (if any) that isn't cancelled. - while waiters: - waiter = waiters.popleft() + # from gh-83055: don't remove future from waiters, + # do it in get() or put() when `waiter` wake up. + for waiter in waiters: if not waiter.done(): waiter.set_result(None) break @@ -111,13 +119,16 @@ async def put(self, item): """Put an item into the queue. Put an item into the queue. If the queue is full, wait until a free - slot is available before adding item. + slot is available before adding item. If there are still item moving + to a free or waiting for free slots, wait for its time. """ - while self.full(): + if self.full() or self._putters: putter = self._get_loop().create_future() self._putters.append(putter) try: await putter + # see gh-83055, clean self._putters. + self._putters.remove(putter) except: putter.cancel() # Just in case putter is not done yet. try: @@ -132,15 +143,26 @@ async def put(self, item): # the call. Wake up the next in line. self._wakeup_next(self._putters) raise - return self.put_nowait(item) + # Don't run tests from self._put_nowait(). + return self._put_and_wakeup_next(item) def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + If there are pending put tasks, raise QueueFullWithPendingPutTasks. """ if self.full(): raise QueueFull + if self._putters: + raise QueueFullWithPendingPutTasks + self._put_and_wakeup_next(item) + + def _put_and_wakeup_next(self, item): + """Put an item into the queue. + + Wakes up next tasks waiting for slots. + """ self._put(item) self._unfinished_tasks += 1 self._finished.clear() @@ -150,12 +172,16 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + If queue is not empty and there are still tasks waiting for items, + task has to wait for its time. """ - while self.empty(): + if self.empty() or self._getters: getter = self._get_loop().create_future() self._getters.append(getter) try: await getter + # see gh-83055, clean self._getters. + self._getters.remove(getter) except: getter.cancel() # Just in case getter is not done yet. try: @@ -170,15 +196,26 @@ async def get(self): # the call. Wake up the next in line. self._wakeup_next(self._getters) raise - return self.get_nowait() + # Don't run tests from self._get_nowait(). + return self._get_and_wakeup_next() def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise QueueEmpty + Return an item if there is no pending get task, else raise QueueEmptyWithPendingGetTasks """ if self.empty(): raise QueueEmpty + if self._getters: + raise QueueEmptyWithPendingGetTasks + return self._get_and_wakeup_next() + + def _get_and_wakeup_next(self): + """Remove and return an item from the queue. + + Wakes up next tasks waiting for slots. + """ item = self._get() self._wakeup_next(self._putters) return item diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2d058ccf6a8c729..cae870a9e4933d8 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -213,6 +213,186 @@ async def test_cancelled_getters_not_being_held_in_self_getters(self): self.assertEqual(len(queue._getters), 0) + async def test_getters_when_put_set_first_future_done(self): + # From this point to all next tests named as 'test_get*' or + # 'test_order*': see gh-83055 + + # Queue empty. + queue = asyncio.Queue(maxsize=1) + + # Simulate queue._getters with already `n' futures. + loop = asyncio.get_running_loop() + n = 2 + queue._getters.extend((loop.create_future() for _ in range(n))) + + # Get one item from queue. + queue.put_nowait(1) + + # After a put() or put_nowait(), no future removed from queue._getters. + self.assertEqual(len(queue._getters), n) + + # First future state set to done. + self.assertTrue(queue._getters[0].done()) + + async def test_getters_not_empty_get_nowait(self): + # Full queue. + queue = asyncio.Queue(1) + queue.put_nowait(1) + + # Simulate queue._getters with already one future. + loop = asyncio.get_running_loop() + queue._getters.append(loop.create_future()) + + # get_nowait() fails. + with self.assertRaises(asyncio.QueueEmptyWithPendingGetTasks): + queue.get_nowait() + + async def test_getters_with_mix_future_states(self): + async def getter(): + await queue.get() + + # Full Queue. + queue = asyncio.Queue(maxsize=4) + # 4 tasks waiting for items. Keep last one in g. + for i in range(4): + g = asyncio.create_task(getter()) + await asyncio.sleep(0) + + # Queue empty, + # 4 tasks waiting for an item. + self.assertTrue(queue.empty()) + self.assertTrue(all(not g.done() for g in queue._getters)) + self.assertEqual(len(queue._getters), 4) + + # Put 2 items, to check if the 2 first futures in queue._getters are `done`. + for i in range(2): + await queue.put(i+1) + + l = list(queue._getters) + # Futures done, reday to get an available item. + self.assertTrue(all(g.done() for g in l[:2])) + # Futures not done, still pending. + self.assertTrue(all(not g.done() for g in l[2:])) + self.assertEqual(len(queue._getters), 4) + + async def test_get_nowait_with_not_empty_getters(self): + # Queue not full. + queue = asyncio.Queue(maxsize=2) + queue.put_nowait(1) + + # Simulate queue._getter with one 'task'. + loop = asyncio.get_running_loop() + waiter = loop.create_future() + queue._getters.append(waiter) + + # Call to get_nowait(). + with self.assertRaises(asyncio.QueueEmptyWithPendingGetTasks): + queue.get_nowait() + + async def test_get_with_not_empty_getters(self): + # Queue not full. + queue = asyncio.Queue(1) + + # Simulate queue._getter with one 'task'. + loop = asyncio.get_running_loop() + waiter = loop.create_future() + queue._getters.append(waiter) + + asyncio.create_task(queue.get()) + await asyncio.sleep(0) + + # Check 2 tasks into queue._getters. + self.assertEqual(len(queue._getters), 2) + + async def test_order_get_nowait_when_exists_free_slot(self): + async def getter(n): + res.append((n, await queue.get())) + + res = [] + # Full queue. + queue = asyncio.Queue(1) + # 3 tasks waiting for items. Saves last one in g. + for i in range(3): + g = asyncio.create_task(getter(i+1)) + await asyncio.sleep(0) + + # Checks queue full and the 2 other items in queue._putters. + self.assertTrue(queue.empty()) + self.assertEqual(len(queue._getters), 3) + + # Put one item, there is now one item available. + # Check first future of _getters is done and elected for the free slot. + # queue._getters always contains 3 futures: one is done, other are not done. + queue.put_nowait(1) + self.assertTrue(queue._getters[0].done()) + self.assertFalse(queue._getters[1].done()) + self.assertFalse(queue._getters[2].done()) + self.assertEqual(len(queue._getters), 3) + self.assertFalse(queue.empty()) + + # Try to stole the item 1 with get_nowait(), + # but raises an exception because queue._getters is not empty. + with self.assertRaises(asyncio.QueueEmptyWithPendingGetTasks): + res.append(queue.get_nowait()) + + # Always 3 futures in queue._getters. + self.assertEqual(len(queue._getters), 3) + + # Put all items. + await queue.put(2) + await queue.put(3) + + # wait for last getter(). + await g + + # Third first items in fifo order, + # 4 item was rejected when called put_nowait(). + self.assertEqual(res, [(1, 1), (2, 2), (3, 3)]) + + async def test_order_get_when_exists_free_slot(self): + async def getter(n): + res.append((n, await queue.get())) + + res = [] + # Full queue. + queue = asyncio.Queue(1) + # 3 tasks waiting for items. + for i in range(3): + asyncio.create_task(getter(i+1)) + await asyncio.sleep(0) + + # Checks queue full and the 2 other items in queue._putters. + self.assertTrue(queue.empty()) + self.assertEqual(len(queue._getters), 3) + + # Put one item, there is now one item available. + # Check first future of _getters is done and elected for the free slot. + # _getters always contains 3 futures: one is done, other are not done. + queue.put_nowait(1) + self.assertTrue(queue._getters[0].done()) + self.assertFalse(queue._getters[1].done()) + self.assertFalse(queue._getters[2].done()) + self.assertEqual(len(queue._getters), 3) + self.assertFalse(queue.empty()) + + # Try to get the item 1, put into queue with put_nowait(), + # task is appended to queue._getters because not empty. + g = asyncio.create_task(getter(4)) + + # Always 3 futures in queue._getters. + self.assertEqual(len(queue._getters), 3) + + # Put all items. + await queue.put(2) + await queue.put(3) + await queue.put(4) + + # wait for last get(). + await g + + # Third first items in fifo order, 4 was never accepted. + self.assertEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + class QueuePutTests(unittest.IsolatedAsyncioTestCase): @@ -403,7 +583,7 @@ async def test_cancelled_puts_not_being_held_in_self_putters(self): await asyncio.sleep(0) # Check that the putter is correctly removed from queue._putters when - # the task is canceled. + # the task is cancelled. self.assertEqual(len(queue._putters), 1) put_task.cancel() with self.assertRaises(asyncio.CancelledError): @@ -431,6 +611,178 @@ async def test_cancelled_put_silence_value_error_exception(self): with self.assertRaises(asyncio.CancelledError): await put_task + async def test_putters_when_get_set_first_future_done(self): + # From this point to all next tests named as 'test_put*' or + # 'test_order*': see gh-83055 + + # Queue full. + queue = asyncio.Queue(maxsize=1) + queue.put_nowait(1) + + # Simulate queue._putters with already 'n' futures. + loop = asyncio.get_running_loop() + n = 2 + queue._putters.extend((loop.create_future() for _ in range(n))) + + # Get one item from queue. + queue.get_nowait() + + # After a get() or get_nowait(), no future removed from queue._putters. + self.assertEqual(len(queue._putters), n) + + # First future state set to done. + self.assertTrue(queue._putters[0].done()) + + async def test_putters_not_empty_get_nowait(self): + # Queue full. + queue = asyncio.Queue(1) + + # Simulate queue._putters with already one future. + loop = asyncio.get_running_loop() + queue._putters.append(loop.create_future()) + + # Put_nowait() fails. + with self.assertRaises(asyncio.QueueFullWithPendingPutTasks): + queue.put_nowait(1) + + async def test_putters_with_mix_future_states(self): + async def putter(item): + await queue.put(item) + + # Full queue. + queue = asyncio.Queue(maxsize=2) + # 4 tasks waiting for items. Keeps last one in g. + for i in range(6): + g = asyncio.create_task(putter(i+1)) + await asyncio.sleep(0) + + # 4 tasks waiting for an item. + self.assertTrue(queue.full()) + self.assertTrue(all(not p.done() for p in queue._putters)) + self.assertEqual(len(queue._putters), 4) + + # Get 2 items, to check if the 2 first futures in queue._getters are `done`. + for i in range(2): + await queue.get() + + l = list(queue._putters) + # 2 futures done, ready to get an available item. + self.assertTrue(all(p.done() for p in l[:2])) + # 2 futures not done, still pending. + self.assertTrue(all(not p.done() for p in l[2:])) + self.assertEqual(len(queue._putters), 4) + + async def test_put_nowait_with_not_empty_putters(self): + # Queue empty. + queue = asyncio.Queue(1) + + # Simulate queue._putters with one 'task'. + loop = asyncio.get_running_loop() + waiter = loop.create_future() + queue._putters.append(waiter) + + # Put an item wit put_nowait(), + # then raises an error + with self.assertRaises(asyncio.QueueFullWithPendingPutTasks): + queue.put_nowait(1) + + async def test_put_with_not_empty_putters(self): + # Queue empty. + queue = asyncio.Queue(1) + + # Simulate queue._putters with one 'task'. + loop = asyncio.get_running_loop() + waiter = loop.create_future() + queue._putters.append(waiter) + + # Add an item into queue. + asyncio.create_task(queue.put(1)) + await asyncio.sleep(0) + + # Check 2 tasks into queue._putters. + self.assertEqual(len(queue._putters), 2) + + async def test_order_put_nowait_when_exists_free_slot(self): + async def putter(item): + await queue.put(item) + + res = [] + # Full queue. + queue = asyncio.Queue(1) + # first item into queue, 2 others items waiting for a free slot. + for i in range(3): + asyncio.create_task(putter(i+1)) + await asyncio.sleep(0) + + # Checks queue full and the 2 other items into queue._putters. + self.assertTrue(queue.full()) + self.assertEqual(len(queue._putters), 2) + + # Get one item, there is now one free slot available. + # Check first future of putters is done and elected for the free slot. + # queue._putters always contains 2 futures: one is done, second is not done + res.append(await queue.get()) + self.assertTrue(queue._putters[0].done()) + self.assertFalse(queue._putters[1].done()) + self.assertFalse(queue.full()) + self.assertEqual(len(queue._putters), 2) + + # Try to stole the free slot with put_nowait(), + # but raises an exception because queue._putters is not empty. + with self.assertRaises(asyncio.QueueFullWithPendingPutTasks): + queue.put_nowait(4) + + # Always 2 futures in queue._putters. + self.assertEqual(len(queue._putters), 2) + # Get all put items. + res.append(await queue.get()) + res.append(await queue.get()) + + # Third first items in fifo order, + # 4 item was put when called put_nowait(). + self.assertEqual(res, [1, 2, 3]) + + async def test_order_put_when_exists_free_slot(self): + async def putter(item): + await queue.put(item) + + res = [] + # Full queue. + queue = asyncio.Queue(1) + # first item into queue, 2 others items waiting for a free slot. + for i in range(3): + asyncio.create_task(putter(i+1)) + await asyncio.sleep(0) + + # Checks queue full and the 2 other items into queue._putters. + self.assertTrue(queue.full()) + self.assertEqual(len(queue._putters), 2) + + # Get one item, there is now one free slot available. + # Check first future of putters is done and elected for the free slot. + # queue?-._putters always contains 2 futures: one is done, second is not done + res.append(await queue.get()) + self.assertTrue(queue._putters[0].done()) + self.assertFalse(queue._putters[1].done()) + self.assertFalse(queue.full()) + self.assertEqual(len(queue._putters), 2) + + # Try to stole the free slot with `put`, + # but tasks appended to an not empty _putters. + asyncio.create_task(putter(4)) + await asyncio.sleep(0) + + # Always 2 futures in _putters. + self.assertEqual(len(queue._putters), 2) + self.assertTrue(queue.full()) + + # Get all put items. + for i in range(3): + res.append(await queue.get()) + + # Checks all items in fifo order. + self.assertEqual(res, [1, 2, 3, 4]) + class LifoQueueTests(unittest.IsolatedAsyncioTestCase):