Logical replication support #42
Next
Next commit
Logical replication
- Loading branch information
commit ca5b5465a07e45f91f6258bfcc73a4dd06ae9e8c
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| # coding: utf-8 | ||
|
|
||
| from six import raise_from | ||
|
|
||
| from .defaults import default_dbname, default_username | ||
| from .exceptions import CatchUpException | ||
| from .utils import pg_version_ge | ||
|
|
||
|
|
||
| class Publication(object): | ||
| def __init__(self, pubname, node, tables=None, dbname=None, username=None): | ||
| """ | ||
| Constructor | ||
|
|
||
| Args: | ||
| pubname: publication name | ||
| node: publisher's node | ||
| tables: tables list or None for all tables | ||
| dbname: database name used to connect and perform subscription | ||
| username: username used to connect to the database | ||
| """ | ||
| self.name = pubname | ||
| self.node = node | ||
| self.dbname = dbname or default_dbname() | ||
| self.username = username or default_username() | ||
|
|
||
| # create publication in database | ||
| t = 'table ' + ', '.join(tables) if tables else 'all tables' | ||
| query = "create publication {} for {}" | ||
| node.safe_psql(query.format(pubname, t), | ||
| dbname=dbname, | ||
| username=username) | ||
|
|
||
| def close(self, dbname=None, username=None): | ||
| """ | ||
| Drop publication | ||
| """ | ||
| self.node.safe_psql("drop publication {}".format(self.name), | ||
| dbname=dbname, username=username) | ||
|
|
||
| def add_tables(self, tables, dbname=None, username=None): | ||
| """ | ||
| Add tables | ||
|
|
||
| Args: | ||
| tables: a list of tables to add to the publication | ||
| """ | ||
| if not tables: | ||
| raise ValueError("Tables list is empty") | ||
|
|
||
| query = "alter publication {} add table {}" | ||
| self.node.safe_psql(query.format(self.name, ', '.join(tables)), | ||
| dbname=dbname or self.dbname, | ||
| username=username or self.username) | ||
|
|
||
|
|
||
| class Subscription(object): | ||
| def __init__(self, | ||
| subname, | ||
| node, | ||
| publication, | ||
| dbname=None, | ||
| username=None, | ||
| **kwargs): | ||
| """ | ||
| Constructor | ||
|
|
||
| Args: | ||
| subname: subscription name | ||
| node: subscriber's node | ||
| publication: Publication object we are subscribing to | ||
| dbname: database name used to connect and perform subscription | ||
| username: username used to connect to the database | ||
| **kwargs: subscription parameters (see CREATE SUBSCRIPTION | ||
| in PostgreSQL documentation for more information) | ||
| """ | ||
| self.name = subname | ||
| self.node = node | ||
| self.pub = publication | ||
|
|
||
| # connection info | ||
| conninfo = ( | ||
| u"dbname={} user={} host={} port={}" | ||
|
There was a problem hiding this comment. I think we could extract common |
||
| ).format(self.pub.dbname, | ||
| self.pub.username, | ||
| self.pub.node.host, | ||
| self.pub.node.port) | ||
|
|
||
| query = ( | ||
| "create subscription {} connection '{}' publication {}" | ||
| ).format(subname, conninfo, self.pub.name) | ||
|
|
||
| # additional parameters | ||
| if kwargs: | ||
| params = ','.join('{}={}'.format(k, v) for k, v in kwargs.iteritems()) | ||
| query += " with ({})".format(params) | ||
|
|
||
| node.safe_psql(query, dbname=dbname, username=username) | ||
|
|
||
| def disable(self, dbname=None, username=None): | ||
| """ | ||
| Disables the running subscription. | ||
| """ | ||
| query = "alter subscription {} disable" | ||
| self.node.safe_psql(query.format(self.name), | ||
| dbname=None, | ||
| username=None) | ||
|
|
||
| def enable(self, dbname=None, username=None): | ||
| """ | ||
| Enables the previously disabled subscription. | ||
| """ | ||
| query = "alter subscription {} enable" | ||
| self.node.safe_psql(query.format(self.name), | ||
| dbname=None, | ||
| username=None) | ||
|
|
||
| def refresh(self, copy_data=True, dbname=None, username=None): | ||
| """ | ||
| Disables the running subscription. | ||
| """ | ||
| query = "alter subscription {} refresh publication with (copy_data={})" | ||
| self.node.safe_psql(query.format(self.name, copy_data), | ||
| dbname=dbname, | ||
| username=username) | ||
|
|
||
| def close(self, dbname=None, username=None): | ||
|
There was a problem hiding this comment. Shouldn't it be called |
||
| """ | ||
| Drops subscription | ||
| """ | ||
| self.node.safe_psql("drop subscription {}".format(self.name), | ||
| dbname=dbname, username=username) | ||
|
|
||
| def catchup(self, username=None): | ||
| """ | ||
| Wait until subscription catches up with publication. | ||
|
|
||
| Args: | ||
| username: remote node's user name | ||
| """ | ||
| if pg_version_ge('10'): | ||
| query = ( | ||
| "select pg_current_wal_lsn() - replay_lsn = 0 " | ||
| "from pg_stat_replication where application_name = '{}'" | ||
| ).format(self.name) | ||
| else: | ||
| query = ( | ||
| "select pg_current_xlog_location() - replay_location = 0 " | ||
| "from pg_stat_replication where application_name = '{}'" | ||
| ).format(self.name) | ||
|
|
||
| try: | ||
| # wait until this LSN reaches subscriber | ||
| self.pub.node.poll_query_until( | ||
| query=query, | ||
| dbname=self.pub.dbname, | ||
| username=username or self.pub.username, | ||
| max_attempts=60, | ||
|
There was a problem hiding this comment. IMHO it's better to replace |
||
| zero_rows_is_ok=True) # statistics may have not updated yet | ||
| except Exception as e: | ||
| raise_from(CatchUpException("Failed to catch up", query), e) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why don't we enable this by default?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Because it is not supported on postgres versions below 10 and there is specific message when someone's trying to enable this feature on those versions. Besides it produces extra WAL data and hence could work slightly slower.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah, i see.