Feature: Add option to force compressed data to be byte aligned

README.md

@@ -210,14 +210,97 @@ The code also allows a wider range of options, including forcing the prediction
 (eg UTF8 vs signed vs MSB vs LSB) and changing the weight of the literal cost from 540
  to other values.
 
-Additionally the CATABLE and APPENDABLE options are exposed and allow concatenation of files
-created in this manner.
+## Stream Concatenation
 
-Specifically CATABLE files can be concatenated in any order using the catbrotli tool
-and APPENDABLE files can be the first file in a sequence of catable files...
-eg you can combine
-appendable.br catable1.br catable2.br catable3.br
+Brotli supports creating streams that can be concatenated together, useful for streaming
+scenarios where you want to compress chunks independently but decompress as a single stream.
 
-or simply
-catable0.br catable1.br catable2.br catable3.br
+### Simple Concatenation (Fast)
 
+Use `-bare -appendable` for the first file and `-bare -catable` for subsequent files.
+These can be combined using plain byte concatenation without special tools, with a
+finalization byte (`0x03`) added at the end:
+
+```bash
+# Create the base file with header but no trailer (must specify window size)
+brotli -c -bare -appendable -w22 input1.txt > base.br
+
+# Create bare-catable streams (no header, no trailer, same window size!)
+brotli -c -bare -catable -w22 input2.txt > part2.br
+brotli -c -bare -catable -w22 input3.txt > part3.br
+
+# Simple concatenation with finalization byte
+# Note: printf '\x03' adds the required final byte
+(cat base.br part2.br part3.br; printf '\x03') > combined.br
+
+# Decompress normally
+brotli -d combined.br -o output.txt
+```
+
+**Advantages:**
+- Instant concatenation (no processing)
+- No special tools required
+- Bare streams can be appended in any order
+
+**Requirements:**
+- All files must use the same window size (`-w22` recommended)
+- First file: `-bare -appendable` (has header, no trailer)
+- Subsequent files: `-bare -catable` (no header, no trailer, no dictionary refs)
+- A final `0x03` byte must be appended to complete the stream
+
+### Efficient Concatenation (Size-optimized)
+
+Use the `catbrotli` tool with `-catable` and `-appendable` flags for better compression
+at the cost of processing time:
+
+```bash
+# Create files for catbrotli tool
+brotli -c -appendable input1.txt > appendable.br
+brotli -c -catable input2.txt > catable1.br
+brotli -c -catable input3.txt > catable2.br
+
+# Concatenate using catbrotli tool
+catbrotli appendable.br catable1.br catable2.br > combined.br
+```
+
+**Tradeoff:** `catbrotli` produces smaller output but requires CPU time to process the
+streams intelligently. Use this when size matters more than concatenation speed.
+
+### Technical Reference: Stream Parameter Interactions
+
+**Stream Types and Their Parameters:**
+
+| Stream Type | bare_stream | byte_align | appendable | catable | use_dictionary | Description |
+|-------------|-------------|------------|------------|---------|----------------|-------------|
+| Standard | false | false | false | false | true | Normal brotli stream with header and trailer |
+| First (simple concat) | true | true | true | false | true | Has header, no trailer - for simple `cat` concatenation |
+| Subsequent (simple concat) | true | true | true | true | false | No header, no trailer, no dict refs - append to first |
+| Appendable (catbrotli) | false | varies | true | false | true | For use with `catbrotli` tool |
+| Catable (catbrotli) | false | varies | true | true | false | For use with `catbrotli` tool |
+
+**Important Notes:**
+- **Parameter dependencies are applied automatically** by the library in both CLI and API usage
+- The library's `SanitizeParams` function ensures:
+  - `catable = true` → automatically sets `appendable = true` and `use_dictionary = false`
+  - `bare_stream = true` → automatically sets `byte_align = true`
+  - `!appendable` → automatically sets `byte_align = false`
+- When using `set_parameter()`, dependencies are applied immediately
+- When setting fields directly (e.g., `params.catable = true`), dependencies are applied during compression initialization
+- **No manual fixups needed** - the library handles all parameter dependencies
+- The `use_dictionary = false` for catable streams prevents references to bytes before the chunk boundary
+- Simple concatenation requires a final `0x03` byte to complete the stream
+- All concatenated streams must use the same window size
+
+**Example API Usage:**
+```rust
+// First file: -bare -appendable equivalent
+params.bare_stream = true;    // Sets bare_stream=true, byte_align=true (automatic)
+params.appendable = true;     // Sets appendable=true
+
+// Subsequent files: -bare -catable equivalent
+params.bare_stream = true;    // Sets bare_stream=true, byte_align=true (automatic)
+params.catable = true;        // Sets catable=true, appendable=true, use_dictionary=false (automatic)
+
+// All parameter dependencies are handled automatically by the library.
+// No manual fixups required - just set the primary flags you want.
+```

c/arg.h

@@ -70,6 +70,21 @@ size_t set_options(BrotliEncoderParameter *out_encoder_param_keys,
             out_encoder_param_values[ret] = 1;
             ret += 1;
         }
+        if (strstr(argv[i], "-appendable") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_APPENDABLE;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
+        if (strstr(argv[i], "-bytealign") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_BYTE_ALIGN;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
+        if (strstr(argv[i], "-bare") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_BARE_STREAM;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
     }
     return ret;
 }

c/brotli/encode.h

@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
   BROTLI_PARAM_APPENDABLE = 168,
-  BROTLI_PARAM_MAGIC_NUMBER = 169
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
+  BROTLI_PARAM_NO_DICTIONARY = 170,
+  BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173
 } BrotliEncoderParameter;
 
 /**

c/go/brotli/brotli.go

@@ -60,6 +60,8 @@ type CompressionOptions struct {
 	Catable                       bool
 	Appendable                    bool
 	Magic                         bool
+	ByteAlign                     bool
+	BareStream                    bool
 	Mode                          int
 	LgWin                         byte
 	LgBlock                       byte
@@ -281,6 +283,14 @@ func makeCompressionOptionsSlices(options CompressionOptions,
 		qualityParams = append(qualityParams, C.BROTLI_PARAM_MAGIC_NUMBER)
 		values = append(values, 1)
 	}
+	if options.ByteAlign {
+		qualityParams = append(qualityParams, C.BROTLI_PARAM_BYTE_ALIGN)
+		values = append(values, 1)
+	}
+	if options.BareStream {
+		qualityParams = append(qualityParams, C.BROTLI_PARAM_BARE_STREAM)
+		values = append(values, 1)
+	}
 	if options.Mode != 0 {
 		qualityParams = append(qualityParams, C.BROTLI_PARAM_MODE)
 		values = append(values, C.uint32_t(options.Mode))

c/go/brotli/brotli/encode.h

@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
   BROTLI_PARAM_APPENDABLE = 168,
-  BROTLI_PARAM_MAGIC_NUMBER = 169
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
+  BROTLI_PARAM_NO_DICTIONARY = 170,
+  BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173,
 } BrotliEncoderParameter;
 
 /**

c/py/brotli.py

@@ -313,6 +313,10 @@ def BrotliParseHeader(raw_data):
 BROTLI_PARAM_CATABLE = 167
 BROTLI_PARAM_APPENDABLE = 168
 BROTLI_PARAM_MAGIC_NUMBER = 169
+BROTLI_PARAM_NO_DICTIONARY = 170
+BROTLI_PARAM_FAVOR_EFFICIENCY = 171
+BROTLI_PARAM_BYTE_ALIGN = 172
+BROTLI_PARAM_BARE_STREAM = 173
 
 #simple test binary
 def main(args):

src/bin/brotli.rs

@@ -553,15 +553,20 @@ fn main() {
             }
             if (argument == "-catable" || argument == "--catable") && !double_dash {
                 params.catable = true;
-                params.use_dictionary = false;
-                params.appendable = true;
                 continue;
             }
             if (argument == "-nothreadpool" || argument == "--nothreadpool") && !double_dash {
                 use_work_pool = false;
                 continue;
             }
-
+            if (argument == "-bytealign" || argument == "--bytealign") && !double_dash {
+                params.byte_align = true;
+                continue;
+            }
+            if (argument == "-bare" || argument == "--bare") && !double_dash {
+                params.bare_stream = true;
+                continue;
+            }
             if (argument == "-appendable" || argument == "--appendable") && !double_dash {
                 params.appendable = true;
                 continue;
@@ -824,7 +829,7 @@ fn main() {
                 continue;
             }
             if argument == "-h" || argument == "-help" || argument == "--help" && !double_dash {
-                println_stderr!("Decompression:\nbrotli [input_file] [output_file]\nCompression:brotli -c -q9.5 -w22 [input_file] [output_file]\nQuality may be one of -q9.5 -q9.5x -q9.5y or -q[0-11] for standard brotli settings.\nOptional size hint -s<size> to direct better compression\n\nThe -i parameter produces a cross human readdable IR representation of the file.\nThis can be ingested by other compressors.\nIR-specific options include:\n-findprior\n-speed=<inc,max,inc,max,inc,max,inc,max>");
+                println_stderr!("Decompression:\nbrotli [input_file] [output_file]\nCompression:brotli -c -q9.5 -w22 [input_file] [output_file]\nQuality may be one of -q9.5 -q9.5x -q9.5y or -q[0-11] for standard brotli settings.\nOptional size hint -s<size> to direct better compression\n\nStream concatenation options:\n-catable     Create stream that can be concatenated with other catable streams\n-appendable  Create stream that can have catable streams appended to it\n-bytealign   Align output to byte boundaries (requires -catable or -appendable)\n-bare        Output bare stream without wrapper (requires -catable or -appendable)\n\nThe -i parameter produces a cross human readdable IR representation of the file.\nThis can be ingested by other compressors.\nIR-specific options include:\n-findprior\n-speed=<inc,max,inc,max,inc,max,inc,max>");
                 return;
             }
             if filenames[0].is_empty() {
@@ -837,7 +842,15 @@ fn main() {
             }
             panic!("Unknown Argument {:}", argument);
         }
-        if !filenames[0].is_empty() {
+        if params.bare_stream && !params.appendable {
+            println_stderr!("bare streams only supported when catable or appendable!");
+            return;
+        }
+        if params.byte_align && !params.appendable {
+            println_stderr!("byte aligned streams only supported when catable or appendable!");
+            return;
+        }
+        if filenames[0] != "" {
             let mut input = match File::open(Path::new(&filenames[0])) {
                 Err(why) => panic!("couldn't open {:}\n{:}", filenames[0], why),
                 Ok(file) => file,