[Issue #174] archive-push improvements: batching, multi-thread support, checksum computation on remote agent, use O_EXCL flag, --no-sync flag support

Author: gsmolk

Makefile

@@ -15,9 +15,9 @@ OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \
 
 EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \
 	src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \
-	src/xlogreader.c
+	src/xlogreader.c src/instr_time.h
 
-INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h
+INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h src/instr_time.h
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -60,6 +60,8 @@ all: checksrcdir $(INCLUDES);
 
 $(PROGRAM): $(OBJS)
 
+src/instr_time.h: $(top_srcdir)/src/include/portability/instr_time.h
+	rm -f $@ && $(LN_S) $(srchome)/src/include/portability/instr_time.h $@
 src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
 	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@
 src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h

doc/pgprobackup.xml

@@ -131,7 +131,6 @@ doc/src/sgml/pgprobackup.sgml
    <arg choice="plain"><option>archive-push</option></arg>
    <arg choice="plain"><option>-B</option> <replaceable>backup_dir</replaceable></arg>
    <arg choice="plain"><option>--instance</option> <replaceable>instance_name</replaceable></arg>
-   <arg choice="plain"><option>--wal-file-path</option> <replaceable>wal_file_path</replaceable></arg>
    <arg choice="plain"><option>--wal-file-name</option> <replaceable>wal_file_name</replaceable></arg>
    <arg rep="repeat"><replaceable>option</replaceable></arg>
   </cmdsynopsis>
@@ -786,7 +785,7 @@ ALTER ROLE backup WITH REPLICATION;
           parameter, as follows:
         </para>
         <programlisting>
-archive_command = '<replaceable>install_dir</replaceable>/pg_probackup archive-push -B <replaceable>backup_dir</replaceable> --instance <replaceable>instance_name</replaceable> --wal-file-path=%p --wal-file-name=%f [<replaceable>remote_options</replaceable>]'
+archive_command = '<replaceable>install_dir</replaceable>/pg_probackup archive-push -B <replaceable>backup_dir</replaceable> --instance <replaceable>instance_name</replaceable> --wal-file-name=%f [<replaceable>remote_options</replaceable>]'
 </programlisting>
       </listitem>
     </itemizedlist>
@@ -3934,9 +3933,12 @@ pg_probackup delete -B <replaceable>backup_dir</replaceable> --instance <replace
       <title>archive-push</title>
       <programlisting>
 pg_probackup archive-push -B <replaceable>backup_dir</replaceable> --instance <replaceable>instance_name</replaceable>
---wal-file-path=<replaceable>wal_file_path</replaceable> --wal-file-name=<replaceable>wal_file_name</replaceable>
-[--help] [--compress] [--compress-algorithm=<replaceable>compression_algorithm</replaceable>]
-[--compress-level=<replaceable>compression_level</replaceable>] [--overwrite]
+--wal-file-name=<replaceable>wal_file_name</replaceable>
+[--help] [--no-sync] [--compress] [--no-ready-rename] [--overwrite]
+[-j <replaceable>num_threads</replaceable>] [--batch-size=<replaceable>batch_size</replaceable>]
+[--archive-timeout=<replaceable>timeout</replaceable>]
+[--compress-algorithm=<replaceable>compression_algorithm</replaceable>]
+[--compress-level=<replaceable>compression_level</replaceable>]
 [<replaceable>remote_options</replaceable>] [<replaceable>logging_options</replaceable>]
 </programlisting>
       <para>
@@ -3961,13 +3963,24 @@ pg_probackup archive-push -B <replaceable>backup_dir</replaceable> --instance <r
         with the <option>--overwrite</option> flag.
       </para>
       <para>
-        The files are copied to a temporary file with the
-        <literal>.part</literal> suffix. After the copy is
-        done, atomic rename is performed. This algorithm ensures that a
-        failed <command>archive-push</command> will not stall continuous archiving and
-        that concurrent archiving from multiple sources into a single
-        WAL archive have no risk of archive corruption. WAL segments copied to
-        the archive are synced to disk.
+        Every file is copied to a temporary file with the
+        <literal>.part</literal> suffix. If the temporary file already
+        exists, <application>pg_probackup</application> will wait
+        <option>archive_timeout</option> seconds before discarding it.
+        After the copy is done, atomic rename is performed.
+        This algorithm ensures that a failed <command>archive-push</command>
+        will not stall continuous archiving and that concurrent archiving from
+        multiple sources into a single WAL archive have no risk of archive
+        corruption.
+      </para>
+      <para>
+        To speed up archiving, especially in remote mode, <command>archive-push</command>
+        can be run on multiple threads using <option>-j num_threads</option> option.
+        Files can also be copied in batches using option <option>--batch-size</option>.
+      </para>
+      <para>
+        WAL segments copied to the archive are synced to disk unless
+        <option>--no-sync</option> flag is used.
       </para>
       <para>
         You can use <command>archive-push</command> in the
@@ -4073,7 +4086,8 @@ pg_probackup archive-get -B <replaceable>backup_dir</replaceable> --instance <re
       <para>
         Sets the number of parallel threads for <command>backup</command>,
         <command>restore</command>, <command>merge</command>,
-        <command>validate</command>, and <command>checkdb</command> processes.
+        <command>validate</command>, <command>checkdb</command> and
+        <command>archive-push</command> processes.
       </para>
       </listitem>
       </varlistentry>
@@ -4732,6 +4746,48 @@ pg_probackup archive-get -B <replaceable>backup_dir</replaceable> --instance <re
       </para>
       </listitem>
       </varlistentry>
+
+      <varlistentry>
+<term><option>--batch-size=batch_size</option></term>
+      <listitem>
+      <para>
+        Sets the maximum number of files to be copied into archive by signle
+        <command>archive-push</command> process.
+      </para>
+      </listitem>
+      </varlistentry>
+
+      <varlistentry>
+<term><option>--archive-timeout=<replaceable>wait_time</replaceable></option></term>
+      <listitem>
+      <para>
+        Sets the timeout for considering existing <literal>.part</literal> file to be stale. By default <application>pg_probackup</application> waits 300 seconds.
+      </para>
+      </listitem>
+      </varlistentry>
+
+      <varlistentry>
+<term><option>--no-ready-rename</option></term>
+      <listitem>
+      <para>
+        Do not rename status files in  <literal>archive_status</literal> directory.
+        This option should be used only if <parameter>archive_command</parameter>
+        contain multiple commands.
+      </para>
+      </listitem>
+      </varlistentry>
+
+      <varlistentry>
+<term><option>--no-sync</option></term>
+      <listitem>
+      <para>
+        Do not sync copied WAL files to disk. You can use this flag to speed
+        up archiving process. Using this flag can result in WAL archive
+        corruption in case of operating system or hardware crash.
+      </para>
+      </listitem>
+      </varlistentry>
+
       </variablelist>
       </para>
     </refsect3>