Merge branch 'master' of git.postgrespro.ru:pgpro-dev/pg_probackup

knizhnik · knizhnik · commit de38db4debb7 · 2019-07-03T23:22:57.000+03:00
diff --git a/Documentation.md b/Documentation.md
@@ -107,7 +107,7 @@ Current version - 2.1.3
 
 As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
 
-- Incremental backup: page-level incremental backup allows you save to disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow
+- Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow
 - Validation: Automatic data consistency checks and on-demand backup validation without actual data recovery
 - Verification: On-demand verification of PostgreSQL instance via dedicated command `checkdb`
 - Retention: Managing backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`
@@ -778,7 +778,7 @@ Specifies remote host user for SSH connection. If you omit this option, the curr
 Specifies pg_probackup installation directory on the remote system.
 
     --ssh-options
-Specifies a string of SSH command-line options.
+Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found here: (https://linux.die.net/man/5/ssh_config)[https://linux.die.net/man/5/ssh_config]
 
 #### Replica Options
 
diff --git a/src/backup.c b/src/backup.c
@@ -344,6 +344,9 @@ do_backup_instance(PGconn *backup_conn)
 			dir_list_file(backup_files_list, parray_get(external_dirs, i),
 						  false, true, false, i+1, FIO_DB_HOST);
 
+	/* close ssh session in main thread */
+	fio_disconnect();
+
 	/* Sanity check for backup_files_list, thank you, Windows:
 	 * https://github.com/postgrespro/pg_probackup/issues/48
 	 */
@@ -512,6 +515,9 @@ do_backup_instance(PGconn *backup_conn)
 		parray_free(prev_backup_filelist);
 	}
 
+	/* Notify end of backup */
+	pg_stop_backup(&current, pg_startbackup_conn);
+
 	/* In case of backup from replica >= 9.6 we must fix minRecPoint,
 	 * First we must find pg_control in backup_files_list.
 	 */
@@ -532,13 +538,16 @@ do_backup_instance(PGconn *backup_conn)
 				break;
 			}
 		}
-	}
 
-	/* Notify end of backup */
-	pg_stop_backup(&current, pg_startbackup_conn);
+		if (!pg_control)
+			elog(ERROR, "Failed to find file \"%s\" in backup filelist.",
+							pg_control_path);
 
-	if (current.from_replica && !exclusive_backup)
 		set_min_recovery_point(pg_control, database_path, current.stop_lsn);
+	}
+
+	/* close ssh session in main thread */
+	fio_disconnect();
 
 	/* Add archived xlog files into the list of files of this backup */
 	if (stream_wal)
@@ -2143,6 +2152,9 @@ backup_files(void *arg)
 			elog(WARNING, "unexpected file type %d", buf.st_mode);
 	}
 
+	/* ssh connection to longer needed */
+	fio_disconnect();
+
 	/* Close connection */
 	if (arguments->conn_arg.conn)
 		pgut_disconnect(arguments->conn_arg.conn);
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -12,8 +12,9 @@
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()
 
-    if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON':
-        loader.testMethodPrefix = 'test_basic'
+    if 'PG_PROBACKUP_TEST_BASIC' in os.environ:
+        if os.environ['PG_PROBACKUP_TEST_BASIC'] == 'ON':
+            loader.testMethodPrefix = 'test_basic'
 
 #    suite.addTests(loader.loadTestsFromModule(auth_test))
     suite.addTests(loader.loadTestsFromModule(archive))
diff --git a/tests/archive.py b/tests/archive.py
@@ -452,22 +452,23 @@ def test_archive_push_partial_file_exists(self):
             filename = filename_orig + '.partial'
             file = os.path.join(wals_dir, filename)
 
+        # emulate stale .partial file
         with open(file, 'a') as f:
             f.write(b"blahblah")
             f.flush()
             f.close()
 
         self.switch_wal_segment(node)
-        sleep(15)
+        sleep(20)
 
         # check that segment is archived
         if self.archive_compress:
             filename_orig = filename_orig + '.gz'
 
         file = os.path.join(wals_dir, filename_orig)
-
         self.assertTrue(os.path.isfile(file))
 
+        # successful validate means that archive-push reused stale wal segment
         self.validate_pb(
             backup_dir, 'node',
             options=['--recovery-target-xid={0}'.format(xid)])
diff --git a/tests/backup.py b/tests/backup.py
@@ -468,16 +468,74 @@ def test_backup_detect_corruption(self):
                 "\n Output: {0} \n CMD: {1}".format(
                     repr(self.output), self.cmd))
         except ProbackupException as e:
-            self.assertIn(
-                'WARNING: Corruption detected in file',
-                e.message,
-                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
-                    repr(e.message), self.cmd))
-            self.assertIn(
-                'ERROR: Data file corruption',
-                e.message,
-                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
-                    repr(e.message), self.cmd))
+            if self.remote:
+                self.assertTrue(
+                    "ERROR: Failed to read file" in e.message and
+                    "data file checksum mismatch" in e.message,
+                    '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                        repr(e.message), self.cmd))
+            else:
+                self.assertIn(
+                    'WARNING: Corruption detected in file',
+                    e.message,
+                    '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                        repr(e.message), self.cmd))
+                self.assertIn(
+                    'ERROR: Data file corruption',
+                    e.message,
+                    '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                        repr(e.message), self.cmd))
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_backup_truncate_misaligned(self):
+        """
+        make node, truncate file to size not even to BLCKSIZE,
+        take backup
+        """
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        node.slow_start()
+
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select 1 as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,100000) i")
+
+        node.safe_psql(
+            "postgres",
+            "CHECKPOINT;")
+
+        heap_path = node.safe_psql(
+            "postgres",
+            "select pg_relation_filepath('t_heap')").rstrip()
+
+        heap_size = node.safe_psql(
+            "postgres",
+            "select pg_relation_size('t_heap')")
+
+        with open(os.path.join(node.data_dir, heap_path), "rb+", 0) as f:
+            f.truncate(int(heap_size) - 4096)
+            f.flush()
+            f.close
+
+        output = self.backup_node(
+            backup_dir, 'node', node, backup_type="full",
+            options=["-j", "4", "--stream"], return_id=False)
+
+        self.assertIn("WARNING: File", output)
+        self.assertIn("invalid file size", output)
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
@@ -1449,16 +1507,16 @@ def test_backup_with_least_privileges_role(self):
 
         node.safe_psql(
             'backupdb',
-            "REVOKE TEMPORARY ON DATABASE backupdb FROM PUBLIC;"
-            "REVOKE ALL on SCHEMA public from PUBLIC; "
+            "REVOKE ALL ON DATABASE backupdb from PUBLIC; "
+            "REVOKE ALL ON SCHEMA public from PUBLIC; "
             "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
             "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
             "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
-            "REVOKE ALL on SCHEMA pg_catalog from PUBLIC; "
+            "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
             "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
             "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
             "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
-            "REVOKE ALL on SCHEMA information_schema from PUBLIC; "
+            "REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
             "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
             "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
             "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
diff --git a/tests/replica.py b/tests/replica.py
@@ -437,3 +437,91 @@ def test_take_backup_from_delayed_replica(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_replica_promote(self):
+        """
+        start backup from replica, during backup promote replica
+        check that backup is failed
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        master = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'master'),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={
+                'archive_timeout': '10s',
+                'checkpoint_timeout': '30s',
+                'max_wal_size': '32MB'})
+
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'master', master)
+        self.set_archiving(backup_dir, 'master', master)
+        master.slow_start()
+
+        replica = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'replica'))
+        replica.cleanup()
+
+        self.backup_node(backup_dir, 'master', master)
+
+        master.psql(
+            "postgres",
+            "create table t_heap as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,165000) i")
+
+        self.restore_node(
+            backup_dir, 'master', replica, options=['-R'])
+
+        # Settings for Replica
+        self.add_instance(backup_dir, 'replica', replica)
+        self.set_archiving(backup_dir, 'replica', replica, replica=True)
+        self.set_replica(
+            master, replica,
+            replica_name='replica', synchronous=True)
+
+        replica.slow_start(replica=True)
+
+        master.psql(
+            "postgres",
+            "create table t_heap_1 as select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0,165000) i")
+
+        self.wait_until_replica_catch_with_master(master, replica)
+
+        # start backup from replica
+        gdb = self.backup_node(
+            backup_dir, 'replica', replica, gdb=True,
+            options=['--log-level-file=verbose'])
+
+        gdb.set_breakpoint('backup_data_file')
+        gdb.run_until_break()
+        gdb.continue_execution_until_break(20)
+
+        replica.promote()
+
+        gdb.remove_all_breakpoints()
+        gdb.continue_execution_until_exit()
+
+        backup_id = self.show_pb(
+            backup_dir, 'replica')[0]["id"]
+
+        # read log file content
+        with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f:
+            log_content = f.read()
+            f.close
+
+        self.assertIn(
+            'ERROR:  the standby was promoted during online backup',
+            log_content)
+
+        self.assertIn(
+            'WARNING: Backup {0} is running, '
+            'setting its status to ERROR'.format(backup_id),
+            log_content)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
