Merge branch 'master' into issue_63

gsmolk · gsmolk · commit 9af4f5ff8156 · 2019-05-06T16:23:19.000+03:00
diff --git a/README.md b/README.md
@@ -43,7 +43,6 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
 `pg_probackup` currently has the following limitations:
 * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number.
 * Remote mode is in beta stage.
-* Microsoft Windows operating system support is in beta stage.
 
 ## Installation and Setup
 ### Linux Installation
diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py
@@ -551,6 +551,35 @@ def check_ptrack_sanity(self, idx_dict):
             #    success, 'Ptrack has failed to register changes in data files'
             # )
 
+    def get_backup_filelist(self, backup_dir, instance, backup_id):
+
+        filelist_path = os.path.join(
+            backup_dir, 'backups',
+            instance, backup_id, 'backup_content.control')
+
+        with open(filelist_path, 'r') as f:
+                filelist_raw = f.read()
+
+        filelist_splitted = filelist_raw.splitlines()
+
+        filelist = {}
+        for line in filelist_splitted:
+            line = json.loads(line)
+            filelist[line['path']] = line
+
+        return filelist
+
+    # return dict of files from filelist A,
+    # which are not exists in filelist_B
+    def get_backup_filelist_diff(self, filelist_A, filelist_B):
+
+        filelist_diff = {}
+        for file in filelist_A:
+            if file not in filelist_B:
+                filelist_diff[file] = filelist_A[file]
+
+        return filelist_diff
+
     def check_ptrack_recovery(self, idx_dict):
         size = idx_dict['size']
         for PageNum in range(size):
diff --git a/tests/restore.py b/tests/restore.py
@@ -1732,11 +1732,12 @@ def test_restore_backup_from_future(self):
         self.backup_node(backup_dir, 'node', node)
 
         node.pgbench_init(scale=3)
-        #pgbench = node.pgbench(options=['-T', '20', '-c', '2'])
-        #pgbench.wait()
+        # pgbench = node.pgbench(options=['-T', '20', '-c', '2'])
+        # pgbench.wait()
 
         # Take PAGE from future
-        backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
+        backup_id = self.backup_node(
+            backup_dir, 'node', node, backup_type='page')
 
         with open(
                 os.path.join(
@@ -1755,7 +1756,8 @@ def test_restore_backup_from_future(self):
         pgbench = node.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum'])
         pgbench.wait()
 
-        backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
+        backup_id = self.backup_node(
+            backup_dir, 'node', node, backup_type='page')
         pgdata = self.pgdata_content(node.data_dir)
 
         node.cleanup()
@@ -1964,7 +1966,8 @@ def test_restore_target_new_options(self):
 
         target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         with node.connect("postgres") as con:
-            res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
+            res = con.execute(
+                "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)")
             con.commit()
             target_xid = res[0][0]
 
@@ -2101,3 +2104,70 @@ def test_restore_target_new_options(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_smart_restore(self):
+        """
+        make node, create database, take full backup, drop database,
+        take incremental backup and restore it,
+        make sure that files from dropped database are not
+        copied during restore
+        https://github.com/postgrespro/pg_probackup/issues/63
+        """
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        # create database
+        node.safe_psql(
+            "postgres",
+            "CREATE DATABASE testdb")
+
+        # take FULL backup
+        full_id = self.backup_node(backup_dir, 'node', node)
+
+        # drop database
+        node.safe_psql(
+            "postgres",
+            "DROP DATABASE testdb")
+
+        # take PAGE backup
+        page_id = self.backup_node(
+            backup_dir, 'node', node, backup_type='page')
+
+        # restore PAGE backup
+        node.cleanup()
+        self.restore_node(
+            backup_dir, 'node', node, backup_id=page_id,
+            options=['--no-validate', '--log-level-file=VERBOSE'])
+
+        logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log')
+        with open(logfile, 'r') as f:
+                logfile_content = f.read()
+
+        # get delta between FULL and PAGE filelists
+        filelist_full = self.get_backup_filelist(
+            backup_dir, 'node', full_id)
+
+        filelist_page = self.get_backup_filelist(
+            backup_dir, 'node', page_id)
+
+        filelist_diff = self.get_backup_filelist_diff(
+            filelist_full, filelist_page)
+
+        for file in filelist_diff:
+            self.assertNotIn(file, logfile_content)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+
+# smart restore of external dirs
