From 46ca7f56a3ab6dec68d7b3bdf119a7fab0db6120 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 14 Jun 2022 12:47:46 -0400 Subject: [PATCH 01/38] Flattened, PBKDF2, Digest, SaltMethod, Python API --- .gitattributes | 1 - .github/workflows/run-bats-core-tests.yml | 2 +- CHANGELOG.md | 11 + contrib/bash/transcrypt | 16 +- contrib/packaging/pacman/PKGBUILD | 2 +- contrib/zsh/_transcrypt | 4 + docs/algorithm.rst | 318 +++++++ example/end_to_end_example.sh | 52 ++ example/legacy_upgrade_example.sh | 45 + sensitive_file | Bin 2560 -> 4416 bytes tests/local_test.sh | 6 + tests/test_init.bats | 6 +- tests/test_transcrypt.py | 501 +++++++++++ tools/fix_indentation.py | 38 + transcrypt | 975 +++++++++++++++------- transcrypt_bashlib.sh | 451 ++++++++++ 16 files changed, 2114 insertions(+), 314 deletions(-) create mode 100644 docs/algorithm.rst create mode 100644 example/end_to_end_example.sh create mode 100644 example/legacy_upgrade_example.sh create mode 100644 tests/local_test.sh create mode 100644 tests/test_transcrypt.py create mode 100644 tools/fix_indentation.py create mode 100644 transcrypt_bashlib.sh diff --git a/.gitattributes b/.gitattributes index 7c479b7..e69de29 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +0,0 @@ -sensitive_file filter=crypt diff=crypt merge=crypt diff --git a/.github/workflows/run-bats-core-tests.yml b/.github/workflows/run-bats-core-tests.yml index 70b9b26..ccc4b76 100644 --- a/.github/workflows/run-bats-core-tests.yml +++ b/.github/workflows/run-bats-core-tests.yml @@ -20,7 +20,7 @@ jobs: - name: Run shellcheck and shfmt uses: luizm/action-sh-checker@master with: - sh_checker_exclude: tests + sh_checker_exclude: "tests example transcrypt_bashlib.sh" sh_checker_comment: true test: diff --git a/CHANGELOG.md b/CHANGELOG.md index de040d1..a6fbce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ The format is based on [Keep a Changelog][1], and this project adheres to [1]: https://keepachangelog.com/en/1.0.0/ [2]: https://semver.org/spec/v2.0.0.html +## [Unreleased] + +### Added + +- Add support for pbkdf2 +- Add support for user specified digest +- Add support for new configured salt method +- Add .transcrypt versioned directory +- Support for OpenSSL 3.x +- Add support for development editable install + ## [2.2.0] - 2022-06-14 ### Added diff --git a/contrib/bash/transcrypt b/contrib/bash/transcrypt index a1096e4..cee7332 100644 --- a/contrib/bash/transcrypt +++ b/contrib/bash/transcrypt @@ -26,13 +26,27 @@ _transcrypt() { case "${prev}" in -c | --cipher) - local ciphers=$(openssl list-cipher-commands) + local ciphers=$(openssl list-cipher-commands || openssl list -cipher-commands &2>/dev/null) COMPREPLY=( $(compgen -W "${ciphers}" -- ${cur}) ) return 0 ;; -p | --password) return 0 ;; + -md | --digest) + local ciphers=$(openssl list-digest-commands || openssl list -digest-commands &2>/dev/null) + COMPREPLY=( $(compgen -W "${ciphers}" -- ${cur}) ) + return 0 + ;; + --use-pbkdf2) + return 0 + ;; + -sm | --salt-method) + return 0 + ;; + -cs | --config-salt) + return 0 + ;; -s | --show-raw) _files_and_dirs return 0 diff --git a/contrib/packaging/pacman/PKGBUILD b/contrib/packaging/pacman/PKGBUILD index 9556e1f..f9b1ef8 100644 --- a/contrib/packaging/pacman/PKGBUILD +++ b/contrib/packaging/pacman/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Aaron Bull Schaefer pkgname=transcrypt -pkgver=2.2.0 +pkgver=3.0.0-pre pkgrel=1 pkgdesc='A script to configure transparent encryption of files within a Git repository' arch=('any') diff --git a/contrib/zsh/_transcrypt b/contrib/zsh/_transcrypt index 7bed4a7..8f8afa5 100644 --- a/contrib/zsh/_transcrypt +++ b/contrib/zsh/_transcrypt @@ -11,6 +11,10 @@ _transcrypt() { '(- 1 *)'{-v,--version}'[print version]' \ '(- 1 *)'{-h,--help}'[view help message]' \ '(-c --cipher -d --display -f --flush-credentials -u --uninstall)'{-c,--cipher=}'[specify encryption cipher]:cipher:->cipher' \ + '(-md --digest -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify encryption digest]:digest' \ + '(-sm --salt-method -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify salt-method]:salt-method' \ + '(-cs --config-salt -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify config-salt]:config-salt' \ + '(-pbkdf2 --use-pbkdf2 -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify use-pbkdf2]:use-pbkdf2' \ '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ '(-d --display -p --password -c --cipher -r --rekey -u --uninstall)'{-d,--display}'[display current credentials]' \ diff --git a/docs/algorithm.rst b/docs/algorithm.rst new file mode 100644 index 0000000..a56b989 --- /dev/null +++ b/docs/algorithm.rst @@ -0,0 +1,318 @@ +The Transcrypt Algorithm +======================== + +The transcrypt algorithm makes use of the following components: + +* `git _` +* `bash _` +* `openssl _` + +The "clean" and "smudge" git filters implement the core functionality by +encrypting a sensitive file before committing it to the repo history, and +decrypting the file when a local copy of the file is checked out. + +* `filter.crypt.clean` - "transcrypt clean" + +* `filter.crypt.smudge` - "transcrypt smudge" + + +Transcrypt uses openssl for all underlying cryptographic operations. + +From git's perspective, is only tracks the encrypted ciphertext of each file. +Thus is it important that any encryption algorithm used must be deterministic, +otherwise changes in the ciphertext (e.g. due to randomized salt) will cause +git to think the file has changed when it hasn't. + + +Core Algorithms +=============== + +From a high level, lets assume we have a secure process to save / load a +desired configuration. + + +The Encryption Process +---------------------- + +A file is encrypted via the following procedure in the ``filter.crypt.clean`` filter. + +Given a sensitive file specified by ``filename`` + +1. Empty files are ignored + +2. A temporary file is created with the (typically plaintext) contents of ``filename``. + This file only contains user read/write permissions (i.e. 600). + A bash trap is set such that this file is removed when transcrypt exists. + +2. The first 6 bytes of the file are checked. If they are "U2FsdGVk" (which is + indicative of a salted openssl encrypted file, we assume the file is already + encrypted emit it as-is) + +3. Otherwise the transcrypt configuration is loaded (which defines the cipher, + digest, key derivation function, salt, and password), openssl is called to + encrypt the plaintext, and the base64 ciphertext is emitted and passed to git. + +The following is the openssl invocation used in encryption + +.. code:: bash + + ENC_PASS=$password openssl enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS -e -a -S "$salt" "${pbkdf2_args[@]}" + + +Note: For OpenSSL V3.x, which does not prepend the salt to the ciphertext, we +manually prepend the raw salt bytes to the raw openssl output (without ``-a`` +for base64 encoding) and then perform base64 encoding of the concatenated text +as a secondary task. This makes the output from version 3.x match outputs from +the 1.x openssl releases. (Also note: this is now independently patched in +https://github.com/elasticdog/transcrypt/pull/135) + + +The Decryption Process +---------------------- + +When a sensitive file is checked out, it is first decrypted before being placed +in the user's working branch via the ``filter.crypt.smudge`` filter. + +1. The ciphertext is passed to the smudge filter via stdin. + +2. The transcrypt configuration is loaded. + +3. The ciphertext is decrypted using openssl and emitted via stdout. If + decryption fails the ciphertext itself is emitted via stdout. + + +The following invocation is used for decryption + +.. code:: bash + + # used to decrypt a file. the cipher, digest, password, and key derivation + # function must be known in advance. the salt is always prepended to the + # file ciphertext, and ready by openssl, so it does not need to be supplied here. + ENC_PASS=$password openssl enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS "${pbkdf2_args[@]}" -d -a + + +Configuration +============= + +Loading the configuration is a critical subroutine in the core transcrypt +components. + +In the proposed transcrypt 3.x implementation, the following *bash* variables +are required for encryption and decryption: + +* ``cipher`` +* ``password`` +* ``digest`` +* ``pbkdf2_args`` + + +And additionally, encryption needs the variable: + +* ``salt`` + +Cipher, Password, and Digest +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For the first 3 variables ``cipher``, ``password``, and ``digest`` the method +transcrypt uses to store them is straightforward. In the local ``.git/config`` +directory these passwords are stored as checkout-specific git variables stored +in plaintext. + +* ``transcrypt.cipher`` +* ``transcrypt.digest`` +* ``transcrypt.password`` + +Note, that before transcrypt 3.x only cipher and password were configurable. +Legacy behavior of transcrypt is described by assuming digest is MD5. + +The other two variables ``pbkdf2_args`` and ``salt`` are less straight forward. + + +PBKDF2 +~~~~~~ + +The `PBKDF2`_ (Password Based Key Derivation Function v2) adds protection +against brute force attacks by increasing the amount of time it takes to derive +the actual key and iv values used in the encryption / decryption process. + +.. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2 + +OpenSSL enables ``pbkdf2`` if the ``-pbkdf2`` flag is specified. +To coerce this into a key-value configuration scheme we use the git +configuration variable + +* ``transcrypt.use-pbkdf2`` + +Which can be set to 0 or 1. At configuration load time, depending on the value +in the config transcrypt will set ``pbkdf2_args`` to an empty bash array in the +case where pbkdf2 is disabled, and ``-pbkdf2`` otherwise. This allows us to use +bash array syntax to express both variants as a single openssl command. + +The backwards compatible setting for transcrypt < 3.x is ``--use-pbkdf2=0``. + +See Also: + +PKCS5#5.2 (RFC-2898) +https://datatracker.ietf.org/doc/html/rfc2898#section-5.2 + +Salt +~~~~ + +Lastly, there is ``salt``, which the least straightforward of these options. + +Ideally, when using openssl, a unique and random salt is generated **each +time** the file is encrypted. This prevents an attacker from executing a +known-plaintext attack by pre-computing common password / ciphertext pairs on +small files and being able to determine the user's password if any of the +precomputed ciphertexts exist in the repo. + +However, transcrypt is unable to use a random salt, because it requires +encryption to be a deterministic process. Otherwise, git would always see a +changed file every time the "clean" command was executed. + +Transcrypt therefore defines two strategies to generate a deterministic salt: + +1. The "password" salt method. +2. The "configured" salt method. + +The first method is equivalent to the existing process in transcrypt 2.x. +The second method is a new more secure variant, but will rely on a new +"versioned config" that we will discuss in +:ref:`the configuration storage section `. + +The two salt methods are very similar. In both cases, a unique 32-byte salt is +generated for each file via the following invocation: + +.. code:: bash + + # Used to compute salt for a specific file using "extra-salt" that can be supplied in one of several ways + openssl dgst -hmac "${filename}:${extra_salt}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16 + +This salt is based on the name of the file, its sha256 hash, and something +called "extra-salt", which is determined by the user's choice of +``transcrypt.salt-method``. + +In the case where ``transcrypt.salt-method=password``, the "extra-salt" is set +to the user's plaintext password. This exactly mimics the behavior of +transcrypt 2.x and is used as the default to provide backwards compatibility. + +However, as discussed in +`#55 _`, this introduces a +security weakness that weakens the extra security provided the use of +``-pbkdf2``. Thus transcrypt 3.x introduces a new "configured" method. + +In the case where ``transcrypt.salt-method=configured``, the implementation +will check if a special configuration variable ``transcrypt.config-salt`` is +set, and if not, it will set it to a random 32 character hex string, and check +the choice of that value into the repo. Then the value of +``transcrypt.config-salt`` will be used as "extra-salt". The value of +``transcrypt.config-salt`` is randomized every time the user changes their +password. We note that this method this method does provide less entropy than +randomly choosing the salt on each encryption cycle, but we are unaware of +any security concerns that arise from this method. + +Note: this method could be further improved by generated a randomized +config-salt for each file that is modified when the file itself is modified. +Such a scheme should exactly match the entropy of the openssl default +randomized salt method. However, due to the added implementation complexity +and unclear security benefits we defer that to future work. + +See Also: + +PKCS5#4.1 (RFC-2898) https://datatracker.ietf.org/doc/html/rfc2898#section-4.1 + +.. _ConfigStorage: + +Configuration Storage +--------------------- + +In transcrypt 2.x, there are currently two ways to store a configuration +containing credentials and + +1. The unversioned config. +2. The GPG-exported config. + +Method 1 stores the configuration in the ``[transcrypt]`` section of the local +``.git/config`` file. This is the primary location for the configuration and +it is typically populated via specifying all settings either via an interactive +process or through non-interactive command line invocation. Whenever transcrypt +is invoked, any needed configuration variable is read from this plaintext file +using git's versatile configuration tool. + +Method 2 is used exclusively for securely transporting configurations between +machines or authorized users. The ``[transcrypt]`` section of an existing +primary configuration in the ``.git/config`` is exported into a simple new line +separated key/value store format, and then encrypted for a specific GPG user. +This encrypted file can be sent to the target recipient. They can then use +transcrypt to "import" the file, which uses +`GPG _` to decrypt the file and +populate their local unversioned ``.git/config`` file. + +In Transcrypt 3.x we propose a third configuration method: + +3. The versioned config. + +Method 3 will store the non-sensitive subset of configuration settings +(everything but ``transcrypt.password``) in a versioned ``.transcrypt/config`` +file using the same git configuration system as Method 1. + +The motivation for this is twofold. + +First, the new deterministic salt method requires a way of storing randomly +sampled bits for the salt (in the ``transcrypt.config-salt`` variable) that are +decorrelated from sensitive information (i.e. the password and contents of +decrypted files). + +Second, transcrypt 3.x adds 4 new parameters that a user will need to +configure. By storing these parameters in the repo itself it will ease the +burden of decrypting a fresh clone of a repo. + +Using this versioned config for everything but ``transcrypt.config-salt`` is +completely optional (and using ``transcrypt.config-salt`` is not needed if +``transcrypt.salt-method=password``, although that is not recommended). Thus +the user can still choose to keep the chosen cipher, digest, and use of pbkdf2 +a secret if they desire (although we will remind the reader that +`security by obscurity _` +should never be relied on). + +NOTE: Currently, as of 2022-05-09, the current implementation of transcrypt 3.x +does not implement the ability for ``.transcrypt/config`` to store any config +variable other than ``transcrypt.config-salt``. We will wait for this proposal +to be reviewed because the design of the priority in which configuration +variables are stored is is currently an open question in the mind of the +author. However, proposed example *behavior* is as follows: + +Case Study and Open Questions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Given: A fresh clone of a repo without a ``.transcrypt/config`` file. + +The user invokes ``transcrypt`` and is prompted for all 6 configuration variables. + +These are stored to the primary ``.git/config`` file, except for +``transcrypt.config-salt``, which --- if the salt method is "configured" --- is +always stored in ``.git/transcrypt`` and checked into the repo. The user is notified +that transcrypt used ``git add`` to stage this file, and instructs the user to commit +the file (transcrypt never invokes the ``git commit`` command). + +Proposal: The user is additionally prompted if they want to add the +non-sensitive configuration to the versioned config. This prompt can be skipped +by specifying ``--versioned=1`` or ``--versioned=0``. In the unversioned case, +the process proceeded as-is, otherwise the non-sensitive configuration is written +to ``.transcrypt/config`` **instead of** being written to ``.git/config``. + +Open Question: When non-sensitive configuration variables are written, should they be: + +1. Written only to ``.transcrypt/config`` and not ``.git/config``? +2. Written to both ``.transcrypt/config`` and ``.git/config``? +3. Written only to ``.transcrypt/config`` and ensured they are removed from ``.git/config``? + +Because all of these configuration files are plain-text and editable we have to +consider the precedence of config settings when loading. The current proposal +is to always look at ``.git/config`` first and then fallback to +``.transcrypt/config``. + +Open Question: When we read a variable from ``.git/config`` and it disagrees +with ``.transcrypt/config`` do we "fix" ``.transcrypt/config``, warn, or ignore +it. My current proposal is to ignore it and rely on documented precedence +rules. diff --git a/example/end_to_end_example.sh b/example/end_to_end_example.sh new file mode 100644 index 0000000..5ea917e --- /dev/null +++ b/example/end_to_end_example.sh @@ -0,0 +1,52 @@ +#!/bin/bash +__doc__=" +A simple demo of transcrypt +" + +TMP_DIR=$HOME/tmp/transcrypt-demo +mkdir -p "$TMP_DIR" +rm -rf "$TMP_DIR" + + +# Make a git repo and add some public content +DEMO_REPO=$TMP_DIR/demo-repo-tc-end-to-end +mkdir -p "$DEMO_REPO" +cd "$DEMO_REPO" +git init +echo "content" > README.md +git add README.md +git commit -m "add readme" + + +# Create safe directory that we will encrypt +echo " +safe/* filter=crypt diff=crypt merge=crypt +" > .gitattributes +git add .gitattributes +git commit -m "add attributes" + +mkdir -p "$DEMO_REPO"/safe + + +# Configure transcrypt with legacy defaults +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --use-pbkdf2=1 -sm random -y +git commit -m "Configured transcrypt" + +echo "Secret contents" > "$DEMO_REPO"/safe/secret_file +cat "$DEMO_REPO"/safe/secret_file + +git add safe/secret_file +git commit -m "add secret with config1" +transcrypt -s safe/secret_file + + +# Rekey with more secure settings +transcrypt --rekey -c aes-256-cbc -p 'foobar' -md SHA256 -y +git commit -am "changed crypto settings" + + +echo "New secret contents" >> "$DEMO_REPO"/safe/secret_file +git commit -am "added secrets" + +transcrypt -d +transcrypt -f -y diff --git a/example/legacy_upgrade_example.sh b/example/legacy_upgrade_example.sh new file mode 100644 index 0000000..5678bfa --- /dev/null +++ b/example/legacy_upgrade_example.sh @@ -0,0 +1,45 @@ +#!/bin/bash +__doc__=" +A simple demo of transcrypt +" + +TMP_DIR=$HOME/tmp/transcrypt-demo +mkdir -p "$TMP_DIR" +rm -rf "$TMP_DIR" + + +# Make a git repo and add some public content +DEMO_REPO=$TMP_DIR/demo-tc-repo-upgrade +mkdir -p "$DEMO_REPO" +cd "$DEMO_REPO" +git init +echo "content" > README.md +git add README.md +git commit -m "add readme" + + +# Create safe directory that we will encrypt +echo " +safe/* filter=crypt diff=crypt merge=crypt +" > .gitattributes +git add .gitattributes +git commit -m "add attributes" + +mkdir -p "$DEMO_REPO"/safe + + +# Configure transcrypt with legacy defaults +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --use-pbkdf2=0 -sm password -y + +echo "Secret contents" > "$DEMO_REPO"/safe/secret_file +cat "$DEMO_REPO"/safe/secret_file + +git add safe/secret_file +git commit -m "add secret with config1" +transcrypt -s safe/secret_file + +echo "New secret contents" >> "$DEMO_REPO"/safe/secret_file +git commit -am "added secrets" + +transcrypt -d +transcrypt -f -y diff --git a/sensitive_file b/sensitive_file index 547ad712f600ee2864e75bf539b829a720b40c0f..a5ab7361c0873fd1e1501b09325a53343b862ec1 100644 GIT binary patch delta 1877 zcmV-b2demh6u=^Xbcw7t47}5{fG21==F~0>LRisY47}y7ej8V9S6-{4GL2Lt01U)O zZF+aYG5AtkL<2+{2O?5|{>oj47NS#&0&g#R=Zxt#3squlpU22+UHek*?DnP1D6qeq zGKULGtr%URD5m{clGC}qFqt>4^Om5DY5oaF?7Em+RAozl=OEUM_1F>ERRB)vg8Z*k zhFJ3WJCwwwaB4CcE@wUxzX3lAXj^Vvge64WqDBh^yPcx7xASq%-S)IW9uQL*a4SzS;B>327P zYM9*DdiwXPU)+liyu|q( zZEvK1Nb8-&XlZi#l1Cj55K0~vx7Dw}!*lXX4_gNMWxw*ArI}tADAv=_XKICOD%(6g z{Y$g|Q3|66hR-0A5|Qry2NBPP3a@)ZPPB&S4%kvG4ad}2V*$bZD79)K91lH4xXLgv zrAao1rc|G;nH$Hz3Ls94mXZ5^jGb$KAbPETM+kpP^(oKEcKGteYCZ;Qwil%z`>g*- z(PHOmbo?%({?3f#c+v1vZ6I^tE_LPfDaSn2;amI9&R@$0q+z~L_*PJM7%zpv6oDoo z#DwSdg<<4&!_v%uz+4arM1pl&y+Xab{X{dy2-QTwSL!)Xlhad4)hS)1gSqgFp%xr} zh(Gq46cj8@>Hk?Bdq-chP0NvH}fu zcmky`5ki7%lUF6Tv)IF1t>&M%!JIivK2djB+4`pPvk9K3%j&w$&*e;!(W4skg}q(3 zbEXa=JW7@;z}vh!lBCiDUzAVa=d!+kbw1g+uN%xB3X0wzZW_dm6nN5={EFKN{9)2so&!7{#!EjKw#6 zT~^aG;gH_B@vIcrf&4-t&ifas6`sf5YoeK~K?dqS@H#~}bm^Oj)oTbmn1wj^->^lR z;s5+Na$mRjH9J*Hj?^i;@ydH~>fa;5vi;PLZgx~C`m3MMqKDvxmV=Z%hFErc;%qxn zz_^zKKAao14_iIBc2dD-2bDyBFYQ-TM#v;bXII%(vx_W3<>P5Z|D|eYX}f=aMm zh%O+09=~0(3b3o=WH6a%sNLK4m%H9t%CYpUlh1^=84Am0y3+ zJc}-@;rdL(c3mdyAKxi+StV;DizZKk%?wDaXNZ?@8_FdWpj6s_cQM|iG=ta2Gx<}w z19};|M-oG2x9}unas3g^8|LjSfR4VwEKR8d77o$WTFk8KB|C+3i741(tB zThb;r2{d>8RkTLRuB@@HLD_pJWkXEfnXR`c!q!CxRoXcZeRAVbiCaW#z-s*I$QmW? z_HQ0ySOIrIDS1JE=7Y9mtq+EeHiR~Eq<)k8+-TUQvY6B#TZD4-|I9PHu5OW}xM+B# zn4&fo`}r(SonUts`LX&sfjt&Fq`m{66!)as_1k6w)7jGE92l{i(_;17JSO=wsG3TD zo>> import sys, ubelt + >>> sys.path.append(ubelt.expandpath('~/code/transcrypt/tests')) + >>> from test_transcrypt import * # NOQA + >>> sandbox = DemoSandbox(verbose=1, dpath='special:cache').setup() + >>> config = {'digest': 'sha256', + >>> 'use_pbkdf2': '1', + >>> 'salt_method': '665896be121e1a0a4a7b18f01780061'} + >>> self = Transcrypt(sandbox.repo_dpath, + >>> config=config, env=sandbox.env, verbose=1) + >>> print(self.version()) + >>> self.config['password'] = 'chbs' + >>> self.login() + >>> sandbox.git.commit('-am', 'new salt config') + >>> print(self.display()) + >>> secret_fpath1 = self.dpath / 'safe/secret1.txt' + >>> secret_fpath2 = self.dpath / 'safe/secret2.txt' + >>> secret_fpath3 = self.dpath / 'safe/secret3.txt' + >>> secret_fpath1.write_text('secret message 1') + >>> secret_fpath2.write_text('secret message 2') + >>> secret_fpath3.write_text('secret message 3') + >>> sandbox.git.add(secret_fpath1, secret_fpath2, secret_fpath3) + >>> sandbox.git.commit('-am', 'add secret messages') + >>> encrypted_paths = self.list() + >>> assert len(encrypted_paths) == 3 + >>> raw_texts = [self.show_raw(p) for p in [secret_fpath1, secret_fpath2, secret_fpath3]] + >>> print('raw_texts = {}'.format(ub.repr2(raw_texts, nl=1))) + >>> assert raw_texts == [ + >>> 'U2FsdGVkX18147KP5UmqOFywveuOGf4hCwrWpfJDp3Ah0HHbFPEGdJE0kM4npWzI', + >>> 'U2FsdGVkX183LEAwwnJ0ne/OKU5VANJsOqCA92Oi9hVkKHIwZYiCgJOoedoShPj7', + >>> 'U2FsdGVkX1/NdLm6twCdF3xYLPCfXacDNsHEeGq0UBC1fwTlJKnN2KmPysS/ylPj', + >>> ] + """ + default_config = { + 'cipher': 'aes-256-cbc', + 'password': None, + 'digest': 'md5', + 'use_pbkdf2': '0', + 'salt_method': 'password', + } + + def __init__(self, dpath, config=None, env=None, transcript_exe=None, verbose=0): + self.dpath = dpath + self.verbose = verbose + self.transcript_exe = ub.Path(ub.find_exe('transcrypt')) + self.env = {} + self.config = self.default_config.copy() + if env is not None: + self.env.update(env) + if config: + self.config.update(config) + + def __nice__(self): + return '{}, {}'.format(self.dpath, ub.repr2(self.config)) + + def _cmd(self, command, shell=False, check=True, verbose=None): + """ + Helper to execute underlying transcrypt commands + """ + if verbose is None: + verbose = self.verbose + return ub.cmd(command, cwd=self.dpath, verbose=verbose, env=self.env, + shell=shell, check=check) + + def _config_args(self): + arg_templates = [ + "-c", self.config['cipher'], + "-p", self.config['password'], + "-md", self.config['digest'], + "--use-pbkdf2", self.config['use_pbkdf2'], + "-sm", self.config['salt_method'], + ] + args = [template.format(**self.config) for template in arg_templates] + return args + + def is_configured(self): + """ + Determine if the transcrypt credentials are populated in the repo + + Returns: + bool : True if the repo is configured with credentials + """ + info = self._cmd(f'{self.transcript_exe} -d', check=0, verbose=0) + return info['ret'] == 0 + + def login(self): + """ + Configure credentials + """ + args = self._config_args() + command = [str(self.transcript_exe), *args, '-y'] + self._cmd(command) + self.config['salt_method'] = self._load_unversioned_config()['salt_method'] + + def logout(self): + """ + Flush credentials + """ + self._cmd(f'{self.transcript_exe} -f -y') + + def rekey(self, new_config): + """ + Re-encrypt all encrypted files using new credentials + """ + self.config.update(new_config) + args = self._config_args() + command = [str(self.transcript_exe), '--rekey', *args, '-y'] + self._cmd(command) + self.config['salt_method'] = self._load_unversioned_config()['salt_method'] + + def display(self): + """ + Returns: + str: the configuration details of the repo + """ + return self._cmd(f'{self.transcript_exe} -d')['out'].rstrip() + + def version(self): + """ + Returns: + str: the version + """ + return self._cmd(f'{self.transcript_exe} --version')['out'].rstrip() + + def _crypt_dir(self): + info = self._cmd('git config --local transcrypt.crypt-dir', check=0) + if info['err'] == 0: + crypt_dpath = ub.Path(info['out'].strip()) + else: + crypt_dpath = self.dpath / '.git/crypt' + return crypt_dpath + + def export_gpg(self, recipient): + """ + Encode the transcrypt credentials securely in an encrypted gpg message + + Returns: + Path: path to the gpg encrypted file containing the repo config + """ + self._cmd(f'{self.transcript_exe} --export-gpg "{recipient}"') + crypt_dpath = self._crypt_dir() + asc_fpath = (crypt_dpath / (recipient + '.asc')) + return asc_fpath + + def import_gpg(self, asc_fpath): + """ + Configure the repo using a given gpg encrypted file + """ + command = f"{self.transcript_exe} --import-gpg '{asc_fpath}' -y" + self._cmd(command) + + def show_raw(self, fpath): + """ + Show the encrypted contents of a file that will be publicly viewable + """ + return self._cmd(f'{self.transcript_exe} -s {fpath}')['out'].rstrip() + + def list(self): + """ + Returns: + List[str]: relative paths of all files managed by transcrypt + """ + result = self._cmd(f'{self.transcript_exe} --list')['out'].rstrip() + paths = result.split('\n') + return paths + + def uninstall(self): + """ + Flushes credentials and removes transcrypt files + """ + return self._cmd(f'{self.transcript_exe} --uninstall -y') + + def upgrade(self): + """ + Upgrades a configured repo to "this" version of transcrypt + """ + return self._cmd(f'{self.transcript_exe} --upgrade -y') + + def _load_unversioned_config(self): + local_config = { + 'cipher': self._cmd('git config --get --local transcrypt.cipher')['out'].strip(), + 'digest': self._cmd('git config --get --local transcrypt.digest')['out'].strip(), + 'use_pbkdf2': self._cmd('git config --get --local transcrypt.use-pbkdf2')['out'].strip(), + 'salt_method': self._cmd('git config --get --local transcrypt.salt-method')['out'].strip(), + 'password': self._cmd('git config --get --local transcrypt.password')['out'].strip(), + 'openssl_path': self._cmd('git config --get --local transcrypt.openssl-path')['out'].strip(), + } + return local_config + + +class DemoSandbox(ub.NiceRepr): + """ + A environment for demo / testing of the transcrypt API + """ + def __init__(self, dpath=None, verbose=0): + if dpath is None: + dpath = 'special:temp' + + if dpath == 'special:temp': + import tempfile + self._tmpdir = tempfile.TemporaryDirectory() + dpath = self._tmpdir.name + elif dpath == 'special:cache': + dpath = ub.Path.appdir('transcrypt/tests/test_env') + self.env = {} + self.dpath = ub.Path(dpath) + self.gpg_store = None + self.repo_dpath = None + self.git = None + self.verbose = verbose + + def __nice__(self): + return str(self.dpath) + + def setup(self): + self._setup_gpghome() + self._setup_gitrepo() + self._setup_contents() + return self + + def _setup_gpghome(self): + if self.verbose: + print('setup sandbox gpghome') + import gpg_lite + self.gpg_home = (self.dpath / 'gpg').ensuredir() + self.gpg_store = gpg_lite.GPGStore( + gnupg_home_dir=self.gpg_home + ) + self.gpg_fpr = self.gpg_store.gen_key( + full_name='Emmy Noether', + email='emmy.noether@uni-goettingen.de', + passphrase=None, + key_type='eddsa', + subkey_type='ecdh', + key_curve='Ed25519', + subkey_curve='Curve25519' + ) + # Fix GNUPG permissions + (self.gpg_home / 'private-keys-v1.d').ensuredir() + # 600 for files and 700 for directories + ub.cmd('find ' + str(self.gpg_home) + r' -type f -exec chmod 600 {} \;', shell=True, cwd=self.gpg_home) + ub.cmd('find ' + str(self.gpg_home) + r' -type d -exec chmod 700 {} \;', shell=True, cwd=self.gpg_home) + self.env['GNUPGHOME'] = str(self.gpg_home) + + def _setup_gitrepo(self): + if self.verbose: + print('setup sandbox gitrepo') + import git + # Make a git repo and add some public content + repo_name = 'demo-repo' + self.repo_dpath = (self.dpath / repo_name).ensuredir() + # self.repo_dpath.delete().ensuredir() + self.repo_dpath.ensuredir() + + for content in self.repo_dpath.iterdir(): + content.delete() + + self.git = git.Git(self.repo_dpath) + self.git.init() + + def _setup_contents(self): + if self.verbose: + print('setup sandbox git contents') + readme_fpath = (self.repo_dpath / 'README.md') + readme_fpath.write_text('content') + self.git.add(readme_fpath) + + # Create safe directory that we will encrypt + gitattr_fpath = self.repo_dpath / '.gitattributes' + gitattr_fpath.write_text(ub.codeblock( + ''' + safe/* filter=crypt diff=crypt merge=crypt + ''')) + self.git.add(gitattr_fpath) + self.git.commit('-am Add initial contents') + self.safe_dpath = (self.repo_dpath / 'safe').ensuredir() + self.secret_fpath = self.safe_dpath / 'secret.txt' + self.secret_fpath.write_text('secret content') + + def _manual_hack_info(self): + """ + Info on how to get an env to run a failing command manually + """ + for k, v in self.env.items(): + print(f'export {k}={v}') + print(f'cd {self.repo_dpath}') + + +class TestCases: + """ + Unit tests to be applied to different transcrypt configurations + """ + + def __init__(self, config=None, dpath=None, verbose=0): + if config is None: + config = Transcrypt.default_config + config['password'] = '12345' + self.config = config + self.verbose = verbose + self.sandbox = None + self.tc = None + self.dpath = dpath + + def setup(self): + self.sandbox = DemoSandbox(dpath=self.dpath, verbose=self.verbose) + self.sandbox.setup() + self.tc = Transcrypt( + dpath=self.sandbox.repo_dpath, + config=self.config, + env=self.sandbox.env, + verbose=self.verbose, + ) + assert not self.tc.is_configured() + self.tc.login() + secret_fpath = self.sandbox.secret_fpath + self.sandbox.git.add(secret_fpath) + self.sandbox.git.commit('-am add secret') + self.tc.display() + + def test_round_trip(self): + secret_fpath = self.sandbox.secret_fpath + ciphertext = self.tc.show_raw(secret_fpath) + plaintext = secret_fpath.read_text() + assert ciphertext.startswith(SALTED_B64) + assert plaintext.startswith('secret content') + assert not plaintext.startswith(SALTED_B64) + + if 0: + print(self.sandbox.git.status()) + self.sandbox.git.diff() + + self.tc.logout() + logged_out_text = secret_fpath.read_text().rstrip() + assert logged_out_text == ciphertext + + self.tc.login() + logged_in_text = secret_fpath.read_text().rstrip() + + assert logged_out_text == ciphertext + assert logged_in_text == plaintext + + def test_export_gpg(self): + self.tc.display() + recipient = self.sandbox.gpg_fpr + asc_fpath = self.tc.export_gpg(recipient) + + info = self.tc._cmd(f'gpg --batch --quiet --decrypt "{asc_fpath}"') + content = info['out'] + got_config = dict([p.split('=', 1) for p in content.split('\n') if p]) + config = self.tc.config.copy() + # FIXME + is_ok = got_config == config + if not is_ok: + is_ok = all([got_config[k] == config[k] for k in {'digest', 'password', 'cipher', 'use_pbkdf2'}]) + + if not is_ok: + print(f'got_config={got_config}') + print(f'config={config}') + raise AssertionError + + assert asc_fpath.exists() + self.tc.logout() + self.tc.import_gpg(asc_fpath) + + secret_fpath = self.sandbox.secret_fpath + plaintext = secret_fpath.read_text() + assert plaintext.startswith('secret content') + + def test_rekey(self): + new_config = { + 'cipher': 'aes-256-cbc', + 'password': '12345', + 'digest': 'sha256', + 'use_pbkdf2': '1', + 'salt_method': 'random', + } + raw_before = self.tc.show_raw(self.sandbox.secret_fpath) + self.tc.rekey(new_config) + self.sandbox.git.commit('-am commit rekey') + raw_after = self.tc.show_raw(self.sandbox.secret_fpath) + assert raw_before != raw_after + + +def test_legacy_defaults(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'md5', + 'use_pbkdf2': '0', + 'salt_method': 'password', + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + self.test_round_trip() + self.test_export_gpg() + + +def test_secure_defaults(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'sha512', + 'use_pbkdf2': '1', + 'salt_method': 'random', + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + self.test_round_trip() + self.test_export_gpg() + + +def test_configured_salt_changes_on_rekey(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'sha512', + 'use_pbkdf2': '1', + 'salt_method': 'random', + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + before_config = self.tc._load_unversioned_config() + self.tc.rekey({'password': '12345', 'salt_method': ''}) + self.sandbox.git.commit('-am commit rekey') + after_config = self.tc._load_unversioned_config() + assert before_config['password'] != after_config['password'] + assert before_config['cipher'] == after_config['cipher'] + assert before_config['use_pbkdf2'] == after_config['use_pbkdf2'] + assert before_config['salt_method'] == after_config['salt_method'] + assert before_config['openssl_path'] == after_config['openssl_path'] + + +def test_configuration_grid(): + """ + CommandLine: + xdoctest -m tests/test_transcrypt.py test_configuration_grid + + Example: + >>> import sys, ubelt + >>> sys.path.append(ubelt.expandpath('~/code/transcrypt/tests')) + >>> from test_transcrypt import * # NOQA + >>> self = TestCases() + >>> self.setup() + >>> self.sandbox._manual_hack_info() + >>> self.test_round_trip() + >>> self.test_export_gpg() + """ + # Test that transcrypt works under a variety of config conditions + basis = { + 'cipher': ['aes-256-cbc', 'aes-128-ecb'], + 'password': ['correct horse battery staple'], + 'digest': ['md5', 'sha256'], + 'use_pbkdf2': ['0', '1'], + 'salt_method': ['password', 'random', 'mylittlecustomsalt'], + } + test_grid = list(ub.named_product(basis)) + verbose = 0 + dpath = 'special:temp' + dpath = 'special:cache' + for params in ub.ProgIter(test_grid, desc='test configs', freq=1): + config = params.copy() + self = TestCases(config=config, dpath=dpath, verbose=verbose) + self.setup() + if 1: + # Manual debug + self.sandbox._manual_hack_info() + + self.test_round_trip() + self.test_export_gpg() + self.test_rekey() + + +if __name__ == '__main__': + """ + CommandLine: + python ~/code/transcrypt/tests/test_transcrypt.py + """ + test_configuration_grid() diff --git a/tools/fix_indentation.py b/tools/fix_indentation.py new file mode 100644 index 0000000..dfd895e --- /dev/null +++ b/tools/fix_indentation.py @@ -0,0 +1,38 @@ +def main(): + import ubelt as ub + import xdev + fpath = ub.Path('$HOME/code/transcrypt/transcrypt').expand() + text = fpath.read_text() + lines = text.split('\n') + + tabstop = 4 + indent_pat = xdev.Pattern.from_regex(r'(\s*)(.*)') + space_pat = xdev.Pattern.from_regex(r' ' * tabstop) + + in_usage = 0 + new_lines = [] + for line in lines: + if 'cat <<-EOF' == line.strip(): + in_usage = True + if 'EOF' == line.strip(): + in_usage = False + indent, suffix = indent_pat.match(line).groups() + hist = ub.dict_hist(indent) + ntabs = hist.get('\t', 0) + if in_usage: + # Only have 2 leading tabs in the usage part + new_indent = space_pat.sub('\t', indent, count=(2 - ntabs)) + else: + new_indent = space_pat.sub('\t', indent) + new_line = new_indent + suffix + new_lines.append(new_line) + + fpath.write_text('\n'.join(new_lines)) + + +if __name__ == '__main__': + """ + CommandLine: + python tools/fix_indentation.py + """ + main() diff --git a/transcrypt b/transcrypt index b392e2d..e43c597 100755 --- a/transcrypt +++ b/transcrypt @@ -1,5 +1,12 @@ #!/usr/bin/env bash -set -euo pipefail +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + # Running as a script + set -euo pipefail +fi + +if [[ "${TRANSCRYPT_TRACE+x}" != "" ]]; then + set -x +fi # # transcrypt - https://github.com/elasticdog/transcrypt @@ -16,10 +23,26 @@ set -euo pipefail ##### CONSTANTS # the release version of this script -readonly VERSION='2.2.0' +readonly VERSION='3.0.0-pre' # the default cipher to utilize readonly DEFAULT_CIPHER='aes-256-cbc' +readonly DEFAULT_DIGEST='MD5' +readonly DEFAULT_USE_PBKDF2='0' +readonly DEFAULT_SALT_METHOD='password' + +# These are the implemented methods for computing deterministic salt +#readonly VALID_SALT_METHODS="password configured random" + +# These are config variables we do not allow to be used in the versioned +# configuration +readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path" + +# Set to 1 to enable a development editable installation +readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=1} + +# Proposed option: can remove or disable later +readonly USE_VERSIONED_CONFIG="0" ##### FUNCTIONS @@ -51,6 +74,208 @@ realpath() { fi } +_openssl_encrypt() { + # In 3.x openssl disabled output of the salt prefix, which we need for determinism. + # We take control over outputting the the prefix 'Salted__' with the salt + # to ensure it is always included regardless of the OpenSSL version. #133 + ( + # Always prepend encrypted ciphertext with "Salted__" prefix and binary salt value + printf "Salted__" && printf "%s" "$salt" | xxd -r -p && + # Encrypt file to binary ciphertext + ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$salt" "${pbkdf2_args[@]}" -in "$tempfile" | + # Strip "Salted__" prefix and salt value if also added by OpenSSL (version < 3) + LC_ALL=C sed -e "s/^\(Salted__.\{8\}\)\(.*\)/\2/" + ) | base64 +} + +_openssl_decrypt() { + # Expects that the following variables are set: + # password, openssl_path, cipher, digest, pbkdf2_args + # This works the same across openssl versions + ENC_PASS=$password "$openssl_path" enc -d "-${cipher}" -md "${digest}" -pass env:ENC_PASS -a "$@" "${pbkdf2_args[@]}" +} + +# compatible openssl list command +_openssl_list() { + arg=$1 + if "${openssl_path}" "list-$arg" &>/dev/null; then + # OpenSSL < v1.1.0 + "${openssl_path}" "list-$arg" + else + # OpenSSL >= v1.1.0 + "${openssl_path}" "list" "-$arg" + fi +} + +# sets a bash global variable by name +_set_global() { + key=$1 + val=$2 + printf -v "$key" '%s' "$val" +} + +# Checks if the target variable is in the set of valid values. If it is not, it +# unsets the target variable, then if not in interactive mode it calls die. +_validate_variable_str() { + local varname=$1 + local valid_values=$2 + local varval=${!varname} + if ! _is_contained_str "$varval" "$valid_values"; then + message=$(printf '%s is ''%s'', but must be one of: %s' "$varname" "$varval" "$valid_values") + if [[ $interactive ]]; then + _set_global "$varname" "" + echo "$message" + else + die 1 "$message" + fi + fi +} + +# Helper to prompt the user, store a response, and validate the result +_get_user_input() { + local varname=$1 + local default=$2 + local validate_fn=$3 + local prompt=$4 + + while [[ ! ${!varname} ]]; do + local answer= + if [[ $interactive ]]; then + printf '%s' "$prompt" + read -r answer + fi + # use the default value if the user gave no answer; otherwise call the + # validate function, which should set the varname to empty if it is + # invalid and the user should continue, otherwise it should die. + if [[ ! $answer ]]; then + _set_global "$varname" "$default" + else + _set_global "$varname" "$answer" + ${validate_fn} + fi + done +} + +# Check if the first arg is contained in the space separated second arg +_is_contained_str() { + arg=$1 + values=$2 + echo "$values" | tr -s ' ' '\n' | grep -Fx "$arg" &>/dev/null +} + +# Load a config var from the versioned config +# shellcheck disable=SC2155 +_load_versioned_config_var() { + # the current git repository's top-level directory + if [ -z "${VERSIONED_TC_CONFIG+x}" ]; then + readonly REPO=$(git rev-parse --show-toplevel 2>/dev/null) + # This is the place where transcrypt can store state that will be + # "versioned" (i.e. checked into the repo) + readonly VERSIONED_TC_DIR="${REPO}/.transcrypt" + readonly VERSIONED_TC_CONFIG="${VERSIONED_TC_DIR}/config" + fi + local key=$1 + # Test for blocked variables that should not go into a plaintext config file + if _is_contained_str "${key}" "${VERSIONED_CONFIG_BLOCKLIST}"; then + warn "Cannot use ${key} in versioned the transcrypt config" + return 1 + fi + git config --file "${VERSIONED_TC_CONFIG}" --get "${key}" || true +} + +# Write a config var to the versioned config +_set_versioned_config_var() { + local key=$1 + local val=$2 + # Test for blocked variables that should not go into a plaintext config file + if _is_contained_str "${key}" "${VERSIONED_CONFIG_BLOCKLIST}"; then + warn "Cannot use ${key} in versioned the transcrypt config" + return 1 + fi + mkdir -p "${VERSIONED_TC_DIR}" + git config --file "${VERSIONED_TC_CONFIG}" "${key}" "${val}" +} + +_set_unversioned_config_var() { + local key=$1 + local val=$2 + #git config --file .git/config "${key}" "${val}" + git config "${key}" "${val}" +} + +_load_unversioned_config_var() { + local key=$1 + git config --get --local "${key}" + #&& (echo "loaded ${key} from .git/config") +} + +# shellcheck disable=SC2181 +_load_config_var() { + # First try loading from the local checkout-independent .git/config file + # If that doesn't work, then look in the .transcrypt/config file + # (which is expected to be stored in plaintext and checked into the repo) + # Certain values will be blocked from being placed here (like the password) + local key=$1 + _load_unversioned_config_var "${key}" + if [[ "$?" != "0" ]]; then + _load_versioned_config_var "${key}" + if [[ "$?" != "0" ]]; then + #(echo "loaded ${key} from versioned config") + true + else + (echo "unable to load ${key}") + fi + else + #(echo "loaded ${key} from unversioned config") + true + fi +} + +_set_config_var() { + # Write to both configs + local key=$1 + local val=$2 + _set_unversioned_config_var "$key" "$val" + if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then + _set_versioned_config_var "$key" "$val" + fi +} + +# shellcheck disable=SC2155 +_load_transcrypt_config_vars() { + # Populate bash vars with our config + cipher=$(_load_config_var "transcrypt.cipher") || (echo "failed to load transcrypt.cipher" && false) + digest=$(_load_config_var "transcrypt.digest") || (echo "failed to load transcrypt.digest" && false) + use_pbkdf2=$(_load_config_var "transcrypt.use-pbkdf2") || (echo "failed to load transcrypt.use-pbkdf2" && false) + salt_method=$(_load_config_var "transcrypt.salt-method") || (echo "failed to load transcrypt.salt-method" && false) + openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) + password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) + ensure_salt_method +} + +_load_vars_for_encryption() { + # Helper to populate variables needed to call openssl encryption + _load_transcrypt_config_vars + + if [[ "$use_pbkdf2" == "1" ]]; then + pbkdf2_args=('-pbkdf2') + else + pbkdf2_args=() + fi + + if [[ "$salt_method" == "password" ]]; then + extra_salt=$password + elif [[ "$salt_method" == "" ]]; then + die "salt must be specified" + else + extra_salt=$salt_method + fi + + if [[ "$extra_salt" == "" ]]; then + die "Extra salt is not set" + fi +} + # establish repository metadata and directory handling # shellcheck disable=SC2155 gather_repo_metadata() { @@ -90,6 +315,13 @@ gather_repo_metadata() { else readonly GIT_ATTRIBUTES="${REPO}/.gitattributes" fi + + # This is the place where transcrypt can store state that will be + # "versioned" (i.e. checked into the repo) + readonly RELATIVE_VERSIONED_TC_DIR=".transcrypt" + readonly RELATIVE_VERSIONED_TC_CONFIG="${RELATIVE_VERSIONED_TC_DIR}/config" + readonly VERSIONED_TC_DIR="${REPO}/${RELATIVE_VERSIONED_TC_DIR}" + readonly VERSIONED_TC_CONFIG="${REPO}/${RELATIVE_VERSIONED_TC_CONFIG}" } # print a message to stderr @@ -116,65 +348,65 @@ die() { # deterministic for everything to work transparently. To do that, the same # salt must be used each time we encrypt the same file. An HMAC has been # proven to be a PRF, so we generate an HMAC-SHA256 for each decrypted file -# (keyed with a combination of the filename and transcrypt password), and +# (keyed with a combination of the filename and ~~transcrypt password~~), and # then use the last 16 bytes of that HMAC for the file's unique salt. - git_clean() { + + # The clean script encrypts files before git sends them to the remote. + # Note the "Salted" check is part of openssl and not anything we do here. + # It allows anyone (including us) to check if a file was already encrypted + # but this does compromise the encrypted stream of data (which starts on + # the 17th byte). + # References: https://crypto.stackexchange.com/questions/8776/what-is-u2fsdgvkx1 filename=$1 # ignore empty files if [[ ! -s $filename ]]; then return fi # cache STDIN to test if it's already encrypted + # First, create the tempfile, then + # set a trap to remove the tempfile when we exit or if anything goes wrong + # finally write the stdin of this script to the tempfile tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) trap 'rm -f "$tempfile"' EXIT tee "$tempfile" &>/dev/null # the first bytes of an encrypted file are always "Salted" in Base64 # The `head + LC_ALL=C tr` command handles binary data in old and new Bash (#116) + # this is an openssl standard. The actual encrypted stream starts on the 17th byte. firstbytes=$(head -c8 "$tempfile" | LC_ALL=C tr -d '\0') if [[ $firstbytes == "U2FsdGVk" ]]; then + # The file is already encrypted, so just pass it back cat "$tempfile" else - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - openssl_path=$(git config --get --local transcrypt.openssl-path) - salt=$("${openssl_path}" dgst -hmac "${filename}:${password}" -sha256 "$tempfile" | tr -d '\r\n' | tail -c16) - - openssl_major_version=$($openssl_path version | cut -d' ' -f2 | cut -d'.' -f1) - if [ "$openssl_major_version" -ge "3" ]; then - # Encrypt the file to base64, ensuring it includes the prefix 'Salted__' with the salt. #133 - ( - echo -n "Salted__" && echo -n "$salt" | xxd -r -p && - # Encrypt file to binary ciphertext - ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md MD5 -pass env:ENC_PASS -S "$salt" -in "$tempfile" - ) | - openssl base64 - else - # Encrypt file to base64 ciphertext - ENC_PASS=$password "$openssl_path" enc -e -a "-${cipher}" -md MD5 -pass env:ENC_PASS -S "$salt" -in "$tempfile" - fi + _load_vars_for_encryption + # NOTE: the openssl standard for salt is 16 hex bytes. + salt=$("$openssl_path" dgst -hmac "${filename}:${extra_salt}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16) + _openssl_encrypt fi } git_smudge() { + # The smudge script decrypts files when they are checked out by an authenticated repository. + # the file contents are passed via stdin tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) trap 'rm -f "$tempfile"' EXIT - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - openssl_path=$(git config --get --local transcrypt.openssl-path) - tee "$tempfile" | ENC_PASS=$password "$openssl_path" enc -d "-${cipher}" -md MD5 -pass env:ENC_PASS -a 2>/dev/null || cat "$tempfile" + #_load_transcrypt_config_vars + _load_vars_for_encryption + tee "$tempfile" | _openssl_decrypt 2>/dev/null || cat "$tempfile" } git_textconv() { + # The textconv script allows users to see git diffs in plaintext. + # It does this by decrypting the encrypted git globs into plain text before + # passing them to the diff command. filename=$1 # ignore empty files if [[ ! -s $filename ]]; then return fi - cipher=$(git config --get --local transcrypt.cipher) - password=$(git config --get --local transcrypt.password) - openssl_path=$(git config --get --local transcrypt.openssl-path) - ENC_PASS=$password "$openssl_path" enc -d "-${cipher}" -md MD5 -pass env:ENC_PASS -a -in "$filename" 2>/dev/null || cat "$filename" + _load_transcrypt_config_vars + _load_vars_for_encryption + _openssl_decrypt -in "$filename" 2>/dev/null || cat "$filename" } # shellcheck disable=SC2005,SC2002,SC2181 @@ -236,6 +468,7 @@ git_pre_commit() { : # Do nothing # The first bytes of an encrypted file must be "Salted" in Base64 elif [[ $firstbytes != "U2FsdGVk" ]]; then + echo "firstbytes = $firstbytes" printf 'Transcrypt managed file is not encrypted in the Git index: %s\n' "$secret_file" >&2 printf '\n' >&2 printf 'You probably staged this file using a tool that does not apply' >&2 @@ -330,48 +563,68 @@ run_safety_checks() { # unset the cipher variable if it is not supported by openssl validate_cipher() { - local list_cipher_commands - if "${openssl_path}" list-cipher-commands &>/dev/null; then - # OpenSSL < v1.1.0 - list_cipher_commands="${openssl_path} list-cipher-commands" - else - # OpenSSL >= v1.1.0 - list_cipher_commands="${openssl_path} list -cipher-commands" - fi + local valid_ciphers + valid_ciphers=$(_openssl_list cipher-commands) + _validate_variable_str "cipher" "$valid_ciphers" +} - local supported - supported=$($list_cipher_commands | tr -s ' ' '\n' | grep -Fx "$cipher") || true - if [[ ! $supported ]]; then - if [[ $interactive ]]; then - printf '"%s" is not a valid cipher; choose one of the following:\n\n' "$cipher" - $list_cipher_commands | column -c 80 - printf '\n' - cipher='' - else - # shellcheck disable=SC2016 - die 1 '"%s" is not a valid cipher; see `%s`' "$cipher" "$list_cipher_commands" - fi +validate_digest() { + local valid_digests + valid_digests=$(_openssl_list digest-commands) + _validate_variable_str "digest" "$valid_digests" +} + +validate_use_pbkdf2() { + _validate_variable_str "use_pbkdf2" "0 1" +} + +validate_salt_method() { + #_validate_variable_str "salt_method" "$VALID_SALT_METHODS" + _="noop" +} + +# ensure we have a digest to hash the salted password +get_digest() { + local prompt + prompt=$(printf 'Encrypt using which digest? [%s] ' "$DEFAULT_DIGEST") + if [[ "$digest" == "" ]]; then + digest=$(_load_versioned_config_var "transcrypt.digest") + # echo "Loaded digest = $digest from local config" fi + _get_user_input digest "$DEFAULT_DIGEST" "validate_digest" "$prompt" } # ensure we have a cipher to encrypt with get_cipher() { - while [[ ! $cipher ]]; do - local answer= - if [[ $interactive ]]; then - printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER" - read -r answer - fi + local prompt + prompt=$(printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER") + if [[ "$cipher" == "" ]]; then + cipher=$(_load_versioned_config_var "transcrypt.cipher") + # echo "Loaded cipher = $cipher from local config" + fi + _get_user_input cipher "$DEFAULT_CIPHER" "validate_cipher" "$prompt" +} - # use the default cipher if the user gave no answer; - # otherwise verify the given cipher is supported by openssl - if [[ ! $answer ]]; then - cipher=$DEFAULT_CIPHER - else - cipher=$answer - validate_cipher - fi - done +get_use_pbkdf2() { + local prompt + prompt=$(printf 'Use pbkdf2? [%s] ' "$DEFAULT_USE_PBKDF2") + if [[ "$use_pbkdf2" == "" ]]; then + use_pbkdf2=$(_load_versioned_config_var "transcrypt.use-pbkdf2") + # echo "Loaded use_pbkdf2 = $use_pbkdf2 from local config" + fi + _get_user_input use_pbkdf2 "$DEFAULT_USE_PBKDF2" "validate_use_pbkdf2" "$prompt" +} + +get_salt_method() { + local prompt + prompt=$(printf 'Compute salt using which method? Can use "password", enter a custom salt, or specify "random". [%s] ' "$DEFAULT_SALT_METHOD") + if [[ "$salt_method" == "" ]]; then + salt_method=$(_load_versioned_config_var "transcrypt.salt-method") + # echo "Loaded salt_method = $salt_method from local config" + fi + _get_user_input salt_method "$DEFAULT_SALT_METHOD" "validate_salt_method" "$prompt" + ensure_salt_method + # echo "Got salt_method = $salt_method" } # ensure we have a password to encrypt with @@ -399,17 +652,28 @@ get_password() { done } +ensure_salt_method() { + # Check if randomized salt needs to be written + if [[ "$salt_method" == "random" ]]; then + # Replace random with something random. + #salt_method=$(_load_versioned_config_var "transcrypt.salt-method") + # If we have not configured the salt_method (or we need to rekey), + # then generate new random salt + salt_method=$(openssl rand -hex 32) + fi + if [[ $rekey ]] && [[ $salt_method != "password" ]]; then + # Assume we want a new random salt unless we are explicitly using password + salt_method=$(openssl rand -hex 32) + fi +} + # confirm the transcrypt configuration confirm_configuration() { local answer= - printf '\nRepository metadata:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" + _display_git_configuration printf 'The following configuration will be saved:\n\n' - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" + _display_runtime_configuration printf 'Does this look correct? [Y/n] ' read -r -n 1 -s answer @@ -426,13 +690,9 @@ confirm_configuration() { confirm_rekey() { local answer= - printf '\nRepository metadata:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" + _display_git_configuration printf 'The following configuration will be saved:\n\n' - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" + _display_runtime_configuration printf 'You are about to re-encrypt all encrypted files using new credentials.\n' printf 'Once you do this, their historical diffs will no longer display in plain text.\n\n' printf 'Proceed with rekey? [y/N] ' @@ -469,8 +729,12 @@ save_helper_scripts() { local current_transcrypt current_transcrypt=$(realpath "$0" 2>/dev/null) - cp "$current_transcrypt" "${CRYPT_DIR}/transcrypt" - + if [[ "$EDITABLE_INSTALL" == "1" ]]; then + # Editable mode is for debugging + ln -fs "$current_transcrypt" "${CRYPT_DIR}/transcrypt" + else + cp -f "$current_transcrypt" "${CRYPT_DIR}/transcrypt" + fi # make scripts executable for script in {transcrypt,}; do chmod 0755 "${CRYPT_DIR}/${script}" @@ -503,16 +767,34 @@ save_helper_hooks() { fi } -# write the configuration to the repository's git config +# "install" transcrypt by writing the configuration to the repository's git config save_configuration() { save_helper_scripts save_helper_hooks # write the encryption info - git config transcrypt.version "$VERSION" - git config transcrypt.cipher "$cipher" - git config transcrypt.password "$password" - git config transcrypt.openssl-path "$openssl_path" + _set_config_var "transcrypt.version" "$VERSION" + _set_config_var "transcrypt.cipher" "$cipher" + _set_config_var "transcrypt.digest" "$digest" + _set_config_var "transcrypt.use-pbkdf2" "$use_pbkdf2" + _set_config_var "transcrypt.salt-method" "$salt_method" + _set_unversioned_config_var "transcrypt.openssl-path" "$openssl_path" + _set_unversioned_config_var "transcrypt.password" "$password" + + if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then + if [ -f $RELATIVE_VERSIONED_TC_CONFIG ]; then + if ! git ls-files --error-unmatch "$RELATIVE_VERSIONED_TC_CONFIG" >/dev/null 2>&1; then + git add "${RELATIVE_VERSIONED_TC_CONFIG}" + printf "*** The contents of %s were configured. ***\n" "${RELATIVE_VERSIONED_TC_CONFIG}" + printf '*** COMMIT THIS CHANGE RIGHT AWAY! ***\n\n' + fi + if ! git diff --exit-code "$RELATIVE_VERSIONED_TC_CONFIG" >/dev/null 2>&1; then + git add "${RELATIVE_VERSIONED_TC_CONFIG}" + printf "*** The contents of %s were updated. ***\n" "${RELATIVE_VERSIONED_TC_CONFIG}" + printf '*** COMMIT THIS CHANGE RIGHT AWAY! ***\n\n' + fi + fi + fi # write the filter settings. Sorry for the horrific quote escaping below... # shellcheck disable=SC2016 @@ -533,23 +815,36 @@ save_configuration() { git config alias.ls-crypt "!git -c core.quotePath=false ls-files | git -c core.quotePath=false check-attr --stdin filter | awk 'BEGIN { FS = \":\" }; /crypt$/{ print \$1 }'" } +_display_git_configuration() { + printf '\nRepository metadata:\n\n' + [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" + printf ' GIT_DIR: %s\n' "$GIT_DIR" + printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" +} + +# Show the config of the current runtime +_display_runtime_configuration() { + printf ' DIGEST: %s\n' "$digest" + printf ' USE_PBKDF2: %s\n' "$use_pbkdf2" + printf ' SALT_METHOD: %s\n' "$salt_method" + if [[ "$salt_method" == "configured" ]]; then + printf ' CONFIG_SALT: %s\n' "$salt_method" + fi + printf ' CIPHER: %s\n' "$cipher" + printf ' PASSWORD: %s\n\n' "$password" +} + # display the current configuration settings display_configuration() { - local current_cipher - current_cipher=$(git config --get --local transcrypt.cipher) - local current_password - current_password=$(git config --get --local transcrypt.password) - local escaped_password=${current_password//\'/\'\\\'\'} - + _load_transcrypt_config_vars + local escaped_password=${password//\'/\'\\\'\'} printf 'The current repository was configured using transcrypt version %s\n' "$CONFIGURED" printf 'and has the following configuration:\n\n' - [[ ! $REPO ]] || printf ' GIT_WORK_TREE: %s\n' "$REPO" - printf ' GIT_DIR: %s\n' "$GIT_DIR" - printf ' GIT_ATTRIBUTES: %s\n\n' "$GIT_ATTRIBUTES" - printf ' CIPHER: %s\n' "$current_cipher" - printf ' PASSWORD: %s\n\n' "$current_password" + _display_git_configuration + _display_runtime_configuration printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - printf " transcrypt -c %s -p '%s'\n" "$current_cipher" "$escaped_password" + printf " transcrypt -c '%s' -p '%s' -md '%s' --use-pbkdf2 '%s' -sm '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$use_pbkdf2" "$salt_method" } # remove transcrypt-related settings from the repository's git config @@ -709,7 +1004,7 @@ uninstall_transcrypt() { sed -i '/filter=crypt diff=crypt merge=crypt[ \t]*$/d' "$GIT_ATTRIBUTES" ;; esac - + # TODO: do we remove .transcrypt/config here? ie rm -f VERSIONED_TC_CONFIG? if [[ ! $upgrade ]]; then printf 'The transcrypt configuration has been completely removed from the repository.\n' fi @@ -819,14 +1114,19 @@ export_gpg() { die 1 'GPG recipient key "%s" does not exist' "$gpg_recipient" fi - local current_cipher - current_cipher=$(git config --get --local transcrypt.cipher) - local current_password - current_password=$(git config --get --local transcrypt.password) + _load_transcrypt_config_vars + + #local current_cipher + #current_cipher=$(git config --get --local transcrypt.cipher) + #local current_password + #current_password=$(git config --get --local transcrypt.password) mkdir -p "${CRYPT_DIR}" local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" - printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" + #printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" + printf 'password=%s\ncipher=%s\ndigest=%s\nuse_pbkdf2=%s\nsalt_method=%s\n\n' \ + "$password" "$cipher" "$digest" "$use_pbkdf2" "$salt_method" | + $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" printf "The transcrypt configuration has been encrypted and exported to:\n%s/crypt/%s.asc\n" "$GIT_DIR" "$gpg_recipient" } @@ -859,6 +1159,10 @@ import_gpg() { cipher=$(printf '%s' "$configuration" | grep '^cipher' | cut -d'=' -f 2-) password=$(printf '%s' "$configuration" | grep '^password' | cut -d'=' -f 2-) + digest=$(printf '%s' "$configuration" | grep '^digest' | cut -d'=' -f 2-) + use_pbkdf2=$(printf '%s' "$configuration" | grep '^use_pbkdf2' | cut -d'=' -f 2-) + salt_method=$(printf '%s' "$configuration" | grep '^salt_method' | cut -d'=' -f 2-) + salt_method=$(printf '%s' "$configuration" | grep '^salt_method' | cut -d'=' -f 2-) } # print this script's usage message to stderr @@ -897,6 +1201,19 @@ help() { the password to derive the key from; defaults to 30 random base64 characters + -md, --digest=DIGEST + the digest used to hash the salted password; + defaults to md5 + + -pbkdf2, --use_pbkdf2=USE_PBKDF2 + Use the pbkdf2 openssl encryption feature; + defaults to 0 + + -sm, --salt_method=SALT_METHOD + Method used to compute deterministic salt; can be password, random, + or a custom string to be used as the salt. Unless set to password, + the salt is randomized on a rekey. + --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH @@ -987,221 +1304,265 @@ help() { } ##### MAIN - -# reset all variables that might be set -cipher='' -display_config='' -flush_creds='' -gpg_import_file='' -gpg_recipient='' -interactive='true' -list='' -password='' -rekey='' -show_file='' -uninstall='' -upgrade='' -openssl_path='openssl' - -# used to bypass certain safety checks -requires_existing_config='' -requires_clean_repo='true' -ignore_config_status='' # Set for operations where config can exist or not - -# parse command line options -while [[ "${1:-}" != '' ]]; do - case $1 in - clean) - shift - git_clean "$@" - exit $? - ;; - smudge) - shift - git_smudge "$@" - exit $? - ;; - textconv) - shift - git_textconv "$@" - exit $? - ;; - merge) - shift - git_merge "$@" - exit $? - ;; - pre_commit) - shift - git_pre_commit "$@" - exit $? - ;; - -c | --cipher) - cipher=$2 - shift - ;; - --cipher=*) - cipher=${1#*=} - ;; - -p | --password) - password=$2 +transcrypt_main() { + # reset all variables that might be set + cipher='' + display_config='' + flush_creds='' + gpg_import_file='' + gpg_recipient='' + interactive='true' + list='' + password='' + rekey='' + show_file='' + uninstall='' + upgrade='' + openssl_path='openssl' + use_pbkdf2='' + digest='' + salt_method='' + salt_method='' + + # used to bypass certain safety checks + requires_existing_config='' + requires_clean_repo='true' + ignore_config_status='' # Set for operations where config can exist or not + + # parse command line options + while [[ "${1:-}" != '' ]]; do + case $1 in + clean) + shift + git_clean "$@" + exit $? + ;; + smudge) + shift + git_smudge "$@" + exit $? + ;; + textconv) + shift + git_textconv "$@" + exit $? + ;; + merge) + shift + git_merge "$@" + exit $? + ;; + pre_commit) + shift + git_pre_commit "$@" + exit $? + ;; + -c | --cipher) + cipher=$2 + shift + ;; + --cipher=*) + cipher=${1#*=} + ;; + -md | --digest) + digest=$2 + shift + ;; + --digest=*) + digest=${1#*=} + ;; + -pbkdf2) + use_pbkdf2=1 + ;; + --use-pbkdf2) + use_pbkdf2=${2} + shift + ;; + --use-pbkdf2=*) + use_pbkdf2=${1#*=} + ;; + -sm | --salt-method) + salt_method=$2 + shift + ;; + --salt-method=*) + salt_method=${1#*=} + ;; + -p | --password) + password=$2 + shift + ;; + --password=*) + password=${1#*=} + ;; + --set-openssl-path=*) + openssl_path=${1#*=} + # Immediately apply config setting + git config transcrypt.openssl-path "$openssl_path" + ;; + -y | --yes) + interactive='' + ;; + -d | --display) + display_config='true' + requires_existing_config='true' + requires_clean_repo='' + ;; + -r | --rekey) + rekey='true' + requires_existing_config='true' + ;; + -f | --flush-credentials) + flush_creds='true' + requires_existing_config='true' + ;; + -F | --force) + requires_clean_repo='' + ;; + -u | --uninstall) + uninstall='true' + requires_existing_config='true' + requires_clean_repo='' + ;; + --upgrade) + upgrade='true' + requires_existing_config='true' + requires_clean_repo='' + ;; + -l | --list) + list='true' + requires_clean_repo='' + ignore_config_status='true' + ;; + -s | --show-raw) + show_file=$2 + show_raw_file + exit 0 + ;; + --show-raw=*) + show_file=${1#*=} + show_raw_file + exit 0 + ;; + -e | --export-gpg) + gpg_recipient=$2 + requires_existing_config='true' + requires_clean_repo='' + shift + ;; + --export-gpg=*) + gpg_recipient=${1#*=} + requires_existing_config='true' + requires_clean_repo='' + ;; + -i | --import-gpg) + gpg_import_file=$2 + shift + ;; + --import-gpg=*) + gpg_import_file=${1#*=} + ;; + -v | --version) + printf 'transcrypt %s\n' "$VERSION" + exit 0 + ;; + -h | --help | -\?) + help + exit 0 + ;; + --*) + warn 'unknown option -- %s' "${1#--}" + usage + exit 1 + ;; + *) + warn 'unknown option -- %s' "${1#-}" + usage + exit 1 + ;; + esac shift - ;; - --password=*) - password=${1#*=} - ;; - --set-openssl-path=*) - openssl_path=${1#*=} - # Immediately apply config setting - git config transcrypt.openssl-path "$openssl_path" - ;; - -y | --yes) - interactive='' - ;; - -d | --display) - display_config='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - -r | --rekey) - rekey='true' - requires_existing_config='true' - ;; - -f | --flush-credentials) - flush_creds='true' - requires_existing_config='true' - ;; - -F | --force) - requires_clean_repo='' - ;; - -u | --uninstall) - uninstall='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - --upgrade) - upgrade='true' - requires_existing_config='true' - requires_clean_repo='' - ;; - -l | --list) - list='true' - requires_clean_repo='' - ignore_config_status='true' - ;; - -s | --show-raw) - show_file=$2 - show_raw_file + done + + gather_repo_metadata + + # always run our safety checks + run_safety_checks + + # regular expression used to test user input + readonly YES_REGEX='^[Yy]$' + + # in order to keep behavior consistent no matter what order the options were + # specified in, we must run these here rather than in the case statement above + if [[ $list ]]; then + list_files exit 0 - ;; - --show-raw=*) - show_file=${1#*=} - show_raw_file + elif [[ $uninstall ]]; then + uninstall_transcrypt exit 0 - ;; - -e | --export-gpg) - gpg_recipient=$2 - requires_existing_config='true' - requires_clean_repo='' - shift - ;; - --export-gpg=*) - gpg_recipient=${1#*=} - requires_existing_config='true' - requires_clean_repo='' - ;; - -i | --import-gpg) - gpg_import_file=$2 - shift - ;; - --import-gpg=*) - gpg_import_file=${1#*=} - ;; - -v | --version) - printf 'transcrypt %s\n' "$VERSION" + elif [[ $upgrade ]]; then + upgrade_transcrypt exit 0 - ;; - -h | --help | -\?) - help + elif [[ $display_config ]] && [[ $flush_creds ]]; then + display_configuration + printf '\n' + flush_credentials exit 0 - ;; - --*) - warn 'unknown option -- %s' "${1#--}" - usage - exit 1 - ;; - *) - warn 'unknown option -- %s' "${1#-}" - usage - exit 1 - ;; - esac - shift -done - -gather_repo_metadata - -# always run our safety checks -run_safety_checks + elif [[ $display_config ]]; then + display_configuration + exit 0 + elif [[ $flush_creds ]]; then + flush_credentials + exit 0 + elif [[ $gpg_recipient ]]; then + export_gpg + exit 0 + elif [[ $gpg_import_file ]]; then + import_gpg + elif [[ $cipher ]]; then + validate_cipher + fi -# regular expression used to test user input -readonly YES_REGEX='^[Yy]$' + # perform function calls to configure transcrypt + get_cipher + get_digest + get_use_pbkdf2 + get_salt_method + get_password + + if [[ $rekey ]] && [[ $interactive ]]; then + confirm_rekey + elif [[ $interactive ]]; then + confirm_configuration + fi -# in order to keep behavior consistent no matter what order the options were -# specified in, we must run these here rather than in the case statement above -if [[ $list ]]; then - list_files - exit 0 -elif [[ $uninstall ]]; then - uninstall_transcrypt - exit 0 -elif [[ $upgrade ]]; then - upgrade_transcrypt - exit 0 -elif [[ $display_config ]] && [[ $flush_creds ]]; then - display_configuration - printf '\n' - flush_credentials - exit 0 -elif [[ $display_config ]]; then - display_configuration - exit 0 -elif [[ $flush_creds ]]; then - flush_credentials - exit 0 -elif [[ $gpg_recipient ]]; then - export_gpg - exit 0 -elif [[ $gpg_import_file ]]; then - import_gpg -elif [[ $cipher ]]; then - validate_cipher -fi + save_configuration -# perform function calls to configure transcrypt -get_cipher -get_password + if [[ $rekey ]]; then + stage_rekeyed_files + else + force_checkout + fi -if [[ $rekey ]] && [[ $interactive ]]; then - confirm_rekey -elif [[ $interactive ]]; then - confirm_configuration -fi + # ensure the git attributes file exists + if [[ ! -f $GIT_ATTRIBUTES ]]; then + mkdir -p "${GIT_ATTRIBUTES%/*}" + printf '#pattern filter=crypt diff=crypt merge=crypt\n' >"$GIT_ATTRIBUTES" + fi -save_configuration + printf 'The repository has been successfully configured by transcrypt.\n' + #exit 0 +} -if [[ $rekey ]]; then - stage_rekeyed_files +# bpkg convention +# https://github.com/bpkg/bpkg +if [[ ${BASH_SOURCE[0]} != "$0" ]]; then + # We are sourcing the library + #export -f transcrypt_main + echo "Sourcing the library" + #export -p else - force_checkout + # Executing file as a script + transcrypt_main "${@}" + exit $? fi - -# ensure the git attributes file exists -if [[ ! -f $GIT_ATTRIBUTES ]]; then - mkdir -p "${GIT_ATTRIBUTES%/*}" - printf '#pattern filter=crypt diff=crypt merge=crypt\n' >"$GIT_ATTRIBUTES" -fi - -printf 'The repository has been successfully configured by transcrypt.\n' - -exit 0 diff --git a/transcrypt_bashlib.sh b/transcrypt_bashlib.sh new file mode 100644 index 0000000..0d588ac --- /dev/null +++ b/transcrypt_bashlib.sh @@ -0,0 +1,451 @@ +#!/usr/bin/env bash +__doc__=' +This contains the standalone heredoc versions of transcrypt library functions. +These are not used in the main executable itself. Instead they are ported from +here to there and stripped of extranious information. + +This makes it easier to unit test the individual bash components of the system +while still providing a fast and reasonably optimized runtime. +' + +# print a message to stderr +warn() { + local fmt="$1" + shift + # shellcheck disable=SC2059 + printf "transcrypt: $fmt\n" "$@" >&2 +} + +# print a message to stderr and exit with either +# the given status or that of the most recent command +die() { + local st="$?" + if [[ "$1" != *[^0-9]* ]]; then + st="$1" + shift + fi + warn "$@" + exit "$st" +} + +# print a canonicalized absolute pathname +realpath() { + local path=$1 + + # make path absolute + local abspath=$path + if [[ -n ${abspath##/*} ]]; then + abspath=$(pwd -P)/$abspath + fi + + # canonicalize path + local dirname= + if [[ -d $abspath ]]; then + dirname=$(cd "$abspath" && pwd -P) + abspath=$dirname + elif [[ -e $abspath ]]; then + dirname=$(cd "${abspath%/*}/" 2>/dev/null && pwd -P) + abspath=$dirname/${abspath##*/} + fi + + if [[ -d $dirname && -e $abspath ]]; then + printf '%s\n' "$abspath" + else + printf 'invalid path: %s\n' "$path" >&2 + exit 1 + fi +} + +joinby(){ + __doc__=' + A function that works similar to a Python join + + Args: + SEP: the separator + *ARR: elements of the strings to join + + Usage: + source $HOME/local/init/utils.sh + ARR=("foo" "bar" "baz") + RESULT=$(joinby / "${ARR[@]}") + echo "RESULT = $RESULT" + + RESULT = foo/bar/baz + + References: + https://stackoverflow.com/questions/1527049/how-can-i-join-elements-of-an-array-in-bash + ' + _handle_help "$@" || return 0 + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi +} + +# shellcheck disable=SC2154 +_openssl_encrypt() +{ + __doc__=' + Example: + source ~/code/transcrypt/transcrypt_bashlib.sh + pbkdf2_args=("-pbkdf2") + salt=deadbeafbad00000 + digest=sha256 + password=12345 + openssl_path=openssl + cipher=aes-256-cbc + tempfile=$(mktemp) + echo "secret" > $tempfile + _openssl_encrypt + ' + # Exepcts that the following variables are set: + # password, openssl_path, cipher, digest, salt, pbkdf2_args, tempfile + + # Test the openssl version + openssl_major_version=$($openssl_path version | cut -d' ' -f2 | cut -d'.' -f1) + if [ "$openssl_major_version" -ge "3" ]; then + # OpenSSL 3.x + # In 3.x openssl disabled output of the salt prefix, which we need for determinism. + # To reenable the prefix we emit the raw prefix bytes, encrypt in raw bytes, and then + # send that entire stream to be base64 encoded + (printf "Salted__" && printf "%s" "$salt" | xxd -r -p && \ + ENC_PASS=$password "$openssl_path" enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS -e -S "$salt" "${pbkdf2_args[@]}" -in "$tempfile" + ) | base64 + else + # OpenSSL 1.x + ENC_PASS=$password "$openssl_path" enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS -e -a -S "$salt" "${pbkdf2_args[@]}" -in "$tempfile" + fi +} + +# shellcheck disable=SC2154 +_openssl_decrypt() +{ + __doc__=' + Example: + source ~/code/transcrypt/transcrypt_bashlib.sh + pbkdf2_args=("-pbkdf2") + digest=sha256 + password=12345 + openssl_path=openssl + cipher=aes-256-cbc + echo "U2FsdGVkX1/erb6vutAAADPXEjWJ3l4MEpSGTj5qC/w=" | _openssl_decrypt + tempfile=$(mktemp) + echo "U2FsdGVkX1/erb6vutAAADPXEjWJ3l4MEpSGTj5qC/w=" > $tempfile + _openssl_decrypt -in $tempfile + ' + # Exepcts that the following variables are set: + # password, openssl_path, cipher, digest, pbkdf2_args + # This works the same across openssl versions + ENC_PASS=$password "$openssl_path" enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS "${pbkdf2_args[@]}" -d -a "$@" +} + +_openssl_list(){ + # Args: the openssl commands to list + __doc__=' + source ~/code/transcrypt/bash_helpers.sh + arg=digest-commands + _openssl_list digest-commands + _openssl_list cipher-commands + ' + openssl_path=openssl + arg=$1 + if "${openssl_path} list-$arg" &>/dev/null; then + # OpenSSL < v1.1.0 + "${openssl_path}" "list-$arg" + else + # OpenSSL >= v1.1.0 + "${openssl_path}" "list" "-$arg" + fi +} + + +_is_contained_str(){ + __doc__=' + Args: + arg : the query to check if it is contained in the values + values : a string of space separated values + + Example: + source ~/code/transcrypt/bash_helpers.sh + # Demo using raw call + (_is_contained_str "foo" "foo bar baz" && echo "contained") || echo "missing" + (_is_contained_str "bar" "foo bar baz" && echo "contained") || echo "missing" + (_is_contained_str "baz" "foo bar baz" && echo "contained") || echo "missing" + (_is_contained_str "biz" "foo bar baz" && echo "contained") || echo "missing" + # Demo using variables + arg="bar" + values="foo bar baz" + (_is_contained_str "$arg" "$values" && echo "contained") || echo "missing" + ' + arg=$1 + values=$2 + echo "$values" | tr -s ' ' '\n' | grep -Fx "$arg" &>/dev/null +} + +_is_contained_arr(){ + __doc__=' + Check if the first value is contained the rest of the values + + Args: + arg : the query to check if it is contained in the values + *values : the rest of the arguments are individual elements in the values + + Example: + source ~/code/transcrypt/bash_helpers.sh + # Demo using raw call + (_is_contained_arr "bar" "foo" "bar" "baz" && echo "contained") || echo "missing" + (_is_contained_arr "biz" "foo" "bar" "baz" && echo "contained") || echo "missing" + # Demo using variables + values=("foo" "bar" "baz") + arg="bar" + (_is_contained_arr "$arg" "${values[@]}" && echo "contained") || echo "missing" + arg="biz" + (_is_contained_arr "$arg" "${values[@]}" && echo "contained") || echo "missing" + ' + # The first argument must be equal to one of the subsequent arguments + local arg=$1 + shift + local arr=("$@") + for val in "${arr[@]}"; + do + if [[ "${arg}" == "${val}" ]]; then + return 0 + fi + done + return 1 +} + +_set_global(){ + # sets a bash global variable by name + key=$1 + val=$2 + printf -v "$key" '%s' "$val" +} + +_validate_variable_arr(){ + __doc__=' + Example: + source bash_helpers.sh + foo="bar" + valid_values=("bar" "biz") + _validate_variable "foo" "${valid_values[@]}" + interactive=1 + _validate_variable "blaz" "${valid_values[@]}" + ' + local varname=$1 + local valid_values=$2 + local varval=${!varname} + if ! _is_contained_arr "$varval" "${valid_values[@]}"; then + local valid_values_str + valid_values_str=$(joinby ', ' "${valid_values[@]}") + message=$(printf "%s is %s, but must be one of: %s" "$varname" "$varval" "$valid_values_str") + if [[ $interactive ]]; then + _set_global "$varname" "" + echo "$message" + else + die 1 "$message" + fi + fi +} + + +_validate_variable_str(){ + __doc__=' + Checks if the target variable is in the set of valid values. + If it is not, it unsets the target variable, then if not in interactive + mode it calls die. + + Args: + varname: name of variable to validate + valid_values: space separated string of valid values + + Example: + source bash_helpers.sh + valid_values="bar biz" + foo="bar" + _validate_variable_str "foo" "$valid_values" + interactive=1 + blaz=fds + _validate_variable_str "blaz" "$valid_values" + ' + local varname=$1 + local valid_values=$2 + local varval=${!varname} + if ! _is_contained_str "$varval" "$valid_values"; then + message=$(printf '%s is `%s`, but must be one of: %s' "$varname" "$varval" "$valid_values") + if [[ $interactive ]]; then + _set_global "$varname" "" + echo "$message" + else + die 1 "$message" + fi + fi +} + +_get_user_input2() { + __doc__=' + Helper to prompt the user, store a response, and validate the result + Args: + varname : name of the bash variable to populate + default : the default value to use if the user provides no answer + valid_values: space separated string of valid values + prompt : string to present to the user + + Example: + source ~/code/transcrypt/bash_helpers.sh + interactive=1 + myvar= + echo "myvar = <$myvar>" + _get_user_input2 "myvar" "a" "a b c" "choose one" + ' + local varname=$1 + local default=$2 + local valid_values=$3 + local prompt=$4 + + while [[ ! ${!varname} ]]; do + local answer= + if [[ $interactive ]]; then + printf '%s > ' "$prompt" + read -r answer + fi + # use the default value if the user gave no answer; otherwise call the + # validate function, which should set the varname to empty if it is + # invalid and the user should continue, otherwise it should die. + if [[ ! $answer ]]; then + _set_global "$varname" "$default" + else + _set_global "$varname" "$answer" + _validate_variable_str "$varname" "$valid_values" + fi + done +} + + +# Helper to prompt the user, store a response, and validate the result +_get_user_input() { + local varname=$1 + local default=$2 + local validate_fn=$3 + local prompt=$4 + + while [[ ! ${!varname} ]]; do + local answer= + if [[ $interactive ]]; then + printf '%s' "$prompt" + read -r answer + fi + # use the default value if the user gave no answer; otherwise call the + # validate function, which should set the varname to empty if it is + # invalid and the user should continue, otherwise it should die. + if [[ ! $answer ]]; then + _set_global "$varname" "$default" + else + _set_global "$varname" "$answer" + ${validate_fn} + fi + done +} + + +# Load a config var from the versioned config +# shellcheck disable=SC2155 +_load_versioned_config_var(){ + # the current git repository's top-level directory + if [ -z "${VERSIONED_TC_CONFIG+x}" ]; then + readonly REPO=$(git rev-parse --show-toplevel 2>/dev/null) + # This is the place where transcrypt can store state that will be + # "versioned" (i.e. checked into the repo) + readonly VERSIONED_TC_DIR="${REPO}/.transcrypt" + readonly VERSIONED_TC_CONFIG="${VERSIONED_TC_DIR}/config" + fi + local key=$1 + # Test for blocked variables that should not go into a plaintext config file + if _is_contained_str "${key}" "${VERSIONED_CONFIG_BLOCKLIST}"; then + warn "Cannot use ${key} in versioned the transcrypt config" + return 1 + fi + # https://unix.stackexchange.com/questions/175648/use-config-file-for-my-shell-script + git config --file "${VERSIONED_TC_CONFIG}" --get "${key}" || true +} + +# Write a config var to the versioned config +_set_versioned_config_var(){ + local key=$1 + local val=$2 + # Test for blocked variables that should not go into a plaintext config file + if _is_contained_str "${key}" "${VERSIONED_CONFIG_BLOCKLIST}"; then + warn "Cannot use ${key} in versioned the transcrypt config" + return 1 + fi + mkdir -p "${VERSIONED_TC_DIR}" + git config --file "${VERSIONED_TC_CONFIG}" "${key}" "${val}" +} + +# +_load_config_var(){ + # First try loading from the local checkout-independent .git/config file + # If that doesn't work, then look in the .transcrypt/config file + # (which is expected to be stored in plaintext and checked into the repo) + # Certain values will be blocked from being placed here (like the password) + local key=$1 + git config --get --local "${key}" + if [[ "$?" != "0" ]]; then + _load_versioned_config_var "${key}" + fi +} + +# shellcheck disable=SC2155 +_load_transcrypt_config_vars(){ + # Populate bash vars with our config + cipher=$(git config --get --local transcrypt.cipher) + digest=$(git config --get --local transcrypt.digest) + use_pbkdf2=$(git config --get --local transcrypt.use-pbkdf2) + salt_method=$(git config --get --local transcrypt.salt-method) + + password=$(git config --get --local transcrypt.password) + openssl_path=$(git config --get --local transcrypt.openssl-path) + + if [[ "$salt_method" == "configured" ]]; then + config_salt=$(_load_versioned_config_var "transcrypt.config-salt") + else + config_salt="" + fi +} + +_load_vars_for_encryption(){ + # Helper to populate variables needed to call openssl encryption + _load_transcrypt_config_vars + + if [[ "$use_pbkdf2" == "1" ]]; then + pbkdf2_args=('-pbkdf2') + else + pbkdf2_args=() + fi + + if [[ "$salt_method" == "password" ]]; then + extra_salt=$password + elif [[ "$salt_method" == "configured" ]]; then + extra_salt=$config_salt + else + die "unknown salt method" + fi + + if [[ "$extra_salt" == "" ]]; then + die "Extra salt is not set" + fi +} + +_benchmark_methods(){ + # Development helepr to determine which way of checking if we have a available digest / cipher is fastest + arg="sha512" + source ~/code/transcrypt/bash_helpers.sh + time (openssl list -digest-commands | tr -s ' ' '\n' | grep -Fx "$arg") + echo $? + time _is_contained_str "$arg" "$(openssl list -digest-commands)" + echo $? + time (readarray -t available <<< "$(openssl list -digest-commands | tr -s ' ' '\n')" && _is_contained_arr "$arg" "${available[@]}") + echo $? + #bash_array_repr "${available[@]}" +} + From 4a595ac153a16c55a41e15c6193e4b7aa69c4453 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 14 Jun 2022 12:52:01 -0400 Subject: [PATCH 02/38] Reset editable install to 0, dont load versioned config when disabled --- transcrypt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/transcrypt b/transcrypt index e43c597..78bb074 100755 --- a/transcrypt +++ b/transcrypt @@ -39,7 +39,7 @@ readonly DEFAULT_SALT_METHOD='password' readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path" # Set to 1 to enable a development editable installation -readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=1} +readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=0} # Proposed option: can remove or disable later readonly USE_VERSIONED_CONFIG="0" @@ -199,14 +199,12 @@ _set_versioned_config_var() { _set_unversioned_config_var() { local key=$1 local val=$2 - #git config --file .git/config "${key}" "${val}" git config "${key}" "${val}" } _load_unversioned_config_var() { local key=$1 git config --get --local "${key}" - #&& (echo "loaded ${key} from .git/config") } # shellcheck disable=SC2181 @@ -218,13 +216,15 @@ _load_config_var() { local key=$1 _load_unversioned_config_var "${key}" if [[ "$?" != "0" ]]; then - _load_versioned_config_var "${key}" - if [[ "$?" != "0" ]]; then - #(echo "loaded ${key} from versioned config") - true - else - (echo "unable to load ${key}") - fi + if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then + _load_versioned_config_var "${key}" + if [[ "$?" != "0" ]]; then + #(echo "loaded ${key} from versioned config") + true + else + (echo "unable to load ${key}") + fi + fi else #(echo "loaded ${key} from unversioned config") true From 0540cb56c21b085d285a0c46b0e7a9eb720053d9 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 14 Jun 2022 13:01:19 -0400 Subject: [PATCH 03/38] Simplified use_pbkdf2 arg to kdf --- tests/test_transcrypt.py | 22 +++++++-------- transcrypt | 60 +++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index fb6105f..77958ef 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -24,7 +24,7 @@ class Transcrypt(ub.NiceRepr): >>> from test_transcrypt import * # NOQA >>> sandbox = DemoSandbox(verbose=1, dpath='special:cache').setup() >>> config = {'digest': 'sha256', - >>> 'use_pbkdf2': '1', + >>> 'kdf': 'pbkdf2', >>> 'salt_method': '665896be121e1a0a4a7b18f01780061'} >>> self = Transcrypt(sandbox.repo_dpath, >>> config=config, env=sandbox.env, verbose=1) @@ -55,7 +55,7 @@ class Transcrypt(ub.NiceRepr): 'cipher': 'aes-256-cbc', 'password': None, 'digest': 'md5', - 'use_pbkdf2': '0', + 'kdf': 'none', 'salt_method': 'password', } @@ -87,7 +87,7 @@ def _config_args(self): "-c", self.config['cipher'], "-p", self.config['password'], "-md", self.config['digest'], - "--use-pbkdf2", self.config['use_pbkdf2'], + "--kdf", self.config['kdf'], "-sm", self.config['salt_method'], ] args = [template.format(**self.config) for template in arg_templates] @@ -200,7 +200,7 @@ def _load_unversioned_config(self): local_config = { 'cipher': self._cmd('git config --get --local transcrypt.cipher')['out'].strip(), 'digest': self._cmd('git config --get --local transcrypt.digest')['out'].strip(), - 'use_pbkdf2': self._cmd('git config --get --local transcrypt.use-pbkdf2')['out'].strip(), + 'kdf': self._cmd('git config --get --local transcrypt.kdf')['out'].strip(), 'salt_method': self._cmd('git config --get --local transcrypt.salt-method')['out'].strip(), 'password': self._cmd('git config --get --local transcrypt.password')['out'].strip(), 'openssl_path': self._cmd('git config --get --local transcrypt.openssl-path')['out'].strip(), @@ -371,7 +371,7 @@ def test_export_gpg(self): # FIXME is_ok = got_config == config if not is_ok: - is_ok = all([got_config[k] == config[k] for k in {'digest', 'password', 'cipher', 'use_pbkdf2'}]) + is_ok = all([got_config[k] == config[k] for k in {'digest', 'password', 'cipher', 'kdf'}]) if not is_ok: print(f'got_config={got_config}') @@ -391,7 +391,7 @@ def test_rekey(self): 'cipher': 'aes-256-cbc', 'password': '12345', 'digest': 'sha256', - 'use_pbkdf2': '1', + 'kdf': 'pbkdf2', 'salt_method': 'random', } raw_before = self.tc.show_raw(self.sandbox.secret_fpath) @@ -406,7 +406,7 @@ def test_legacy_defaults(): 'cipher': 'aes-256-cbc', 'password': 'correct horse battery staple', 'digest': 'md5', - 'use_pbkdf2': '0', + 'kdf': 'none', 'salt_method': 'password', } verbose = 1 @@ -421,7 +421,7 @@ def test_secure_defaults(): 'cipher': 'aes-256-cbc', 'password': 'correct horse battery staple', 'digest': 'sha512', - 'use_pbkdf2': '1', + 'kdf': 'pbkdf2', 'salt_method': 'random', } verbose = 1 @@ -436,7 +436,7 @@ def test_configured_salt_changes_on_rekey(): 'cipher': 'aes-256-cbc', 'password': 'correct horse battery staple', 'digest': 'sha512', - 'use_pbkdf2': '1', + 'kdf': 'pbkdf2', 'salt_method': 'random', } verbose = 1 @@ -448,7 +448,7 @@ def test_configured_salt_changes_on_rekey(): after_config = self.tc._load_unversioned_config() assert before_config['password'] != after_config['password'] assert before_config['cipher'] == after_config['cipher'] - assert before_config['use_pbkdf2'] == after_config['use_pbkdf2'] + assert before_config['kdf'] == after_config['kdf'] assert before_config['salt_method'] == after_config['salt_method'] assert before_config['openssl_path'] == after_config['openssl_path'] @@ -473,7 +473,7 @@ def test_configuration_grid(): 'cipher': ['aes-256-cbc', 'aes-128-ecb'], 'password': ['correct horse battery staple'], 'digest': ['md5', 'sha256'], - 'use_pbkdf2': ['0', '1'], + 'kdf': ['none', 'pbkdf2'], 'salt_method': ['password', 'random', 'mylittlecustomsalt'], } test_grid = list(ub.named_product(basis)) diff --git a/transcrypt b/transcrypt index 78bb074..e40c1e7 100755 --- a/transcrypt +++ b/transcrypt @@ -28,7 +28,7 @@ readonly VERSION='3.0.0-pre' # the default cipher to utilize readonly DEFAULT_CIPHER='aes-256-cbc' readonly DEFAULT_DIGEST='MD5' -readonly DEFAULT_USE_PBKDF2='0' +readonly DEFAULT_KDF='none' readonly DEFAULT_SALT_METHOD='password' # These are the implemented methods for computing deterministic salt @@ -246,7 +246,7 @@ _load_transcrypt_config_vars() { # Populate bash vars with our config cipher=$(_load_config_var "transcrypt.cipher") || (echo "failed to load transcrypt.cipher" && false) digest=$(_load_config_var "transcrypt.digest") || (echo "failed to load transcrypt.digest" && false) - use_pbkdf2=$(_load_config_var "transcrypt.use-pbkdf2") || (echo "failed to load transcrypt.use-pbkdf2" && false) + kdf=$(_load_config_var "transcrypt.kdf") || (echo "failed to load transcrypt.kdf" && false) salt_method=$(_load_config_var "transcrypt.salt-method") || (echo "failed to load transcrypt.salt-method" && false) openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) @@ -257,7 +257,7 @@ _load_vars_for_encryption() { # Helper to populate variables needed to call openssl encryption _load_transcrypt_config_vars - if [[ "$use_pbkdf2" == "1" ]]; then + if [[ "$kdf" == "1" ]] || [[ "$kdf" == "pbkdf2" ]]; then pbkdf2_args=('-pbkdf2') else pbkdf2_args=() @@ -574,8 +574,8 @@ validate_digest() { _validate_variable_str "digest" "$valid_digests" } -validate_use_pbkdf2() { - _validate_variable_str "use_pbkdf2" "0 1" +validate_kdf() { + _validate_variable_str "kdf" "0 1 none pbkdf2" } validate_salt_method() { @@ -605,14 +605,14 @@ get_cipher() { _get_user_input cipher "$DEFAULT_CIPHER" "validate_cipher" "$prompt" } -get_use_pbkdf2() { +get_kdf() { local prompt - prompt=$(printf 'Use pbkdf2? [%s] ' "$DEFAULT_USE_PBKDF2") - if [[ "$use_pbkdf2" == "" ]]; then - use_pbkdf2=$(_load_versioned_config_var "transcrypt.use-pbkdf2") - # echo "Loaded use_pbkdf2 = $use_pbkdf2 from local config" + prompt=$(printf 'Which key derivation function? [%s] ' "$DEFAULT_KDF") + if [[ "$kdf" == "" ]]; then + kdf=$(_load_versioned_config_var "transcrypt.kdf") + # echo "Loaded kdf = $kdf from local config" fi - _get_user_input use_pbkdf2 "$DEFAULT_USE_PBKDF2" "validate_use_pbkdf2" "$prompt" + _get_user_input kdf "$DEFAULT_KDF" "validate_kdf" "$prompt" } get_salt_method() { @@ -656,7 +656,6 @@ ensure_salt_method() { # Check if randomized salt needs to be written if [[ "$salt_method" == "random" ]]; then # Replace random with something random. - #salt_method=$(_load_versioned_config_var "transcrypt.salt-method") # If we have not configured the salt_method (or we need to rekey), # then generate new random salt salt_method=$(openssl rand -hex 32) @@ -776,7 +775,7 @@ save_configuration() { _set_config_var "transcrypt.version" "$VERSION" _set_config_var "transcrypt.cipher" "$cipher" _set_config_var "transcrypt.digest" "$digest" - _set_config_var "transcrypt.use-pbkdf2" "$use_pbkdf2" + _set_config_var "transcrypt.kdf" "$kdf" _set_config_var "transcrypt.salt-method" "$salt_method" _set_unversioned_config_var "transcrypt.openssl-path" "$openssl_path" _set_unversioned_config_var "transcrypt.password" "$password" @@ -825,7 +824,7 @@ _display_git_configuration() { # Show the config of the current runtime _display_runtime_configuration() { printf ' DIGEST: %s\n' "$digest" - printf ' USE_PBKDF2: %s\n' "$use_pbkdf2" + printf ' USE_PBKDF2: %s\n' "$kdf" printf ' SALT_METHOD: %s\n' "$salt_method" if [[ "$salt_method" == "configured" ]]; then printf ' CONFIG_SALT: %s\n' "$salt_method" @@ -843,8 +842,8 @@ display_configuration() { _display_git_configuration _display_runtime_configuration printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - printf " transcrypt -c '%s' -p '%s' -md '%s' --use-pbkdf2 '%s' -sm '%s'\n" \ - "$cipher" "$escaped_password" "$digest" "$use_pbkdf2" "$salt_method" + printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -sm '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$kdf" "$salt_method" } # remove transcrypt-related settings from the repository's git config @@ -1124,8 +1123,8 @@ export_gpg() { local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" #printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" - printf 'password=%s\ncipher=%s\ndigest=%s\nuse_pbkdf2=%s\nsalt_method=%s\n\n' \ - "$password" "$cipher" "$digest" "$use_pbkdf2" "$salt_method" | + printf 'password=%s\ncipher=%s\ndigest=%s\nkdf=%s\nsalt_method=%s\n\n' \ + "$password" "$cipher" "$digest" "$kdf" "$salt_method" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" printf "The transcrypt configuration has been encrypted and exported to:\n%s/crypt/%s.asc\n" "$GIT_DIR" "$gpg_recipient" } @@ -1160,7 +1159,7 @@ import_gpg() { cipher=$(printf '%s' "$configuration" | grep '^cipher' | cut -d'=' -f 2-) password=$(printf '%s' "$configuration" | grep '^password' | cut -d'=' -f 2-) digest=$(printf '%s' "$configuration" | grep '^digest' | cut -d'=' -f 2-) - use_pbkdf2=$(printf '%s' "$configuration" | grep '^use_pbkdf2' | cut -d'=' -f 2-) + kdf=$(printf '%s' "$configuration" | grep '^kdf' | cut -d'=' -f 2-) salt_method=$(printf '%s' "$configuration" | grep '^salt_method' | cut -d'=' -f 2-) salt_method=$(printf '%s' "$configuration" | grep '^salt_method' | cut -d'=' -f 2-) } @@ -1205,9 +1204,12 @@ help() { the digest used to hash the salted password; defaults to md5 - -pbkdf2, --use_pbkdf2=USE_PBKDF2 - Use the pbkdf2 openssl encryption feature; - defaults to 0 + --kdf=PBKDF2 + the key-derivation-function to use. Currently can be either + 'pbkdf2' or 'none'. Defaults to none. + + -pbkdf2 + equivalent to passing --kdf2='pbkdf2' -sm, --salt_method=SALT_METHOD Method used to compute deterministic salt; can be password, random, @@ -1319,7 +1321,7 @@ transcrypt_main() { uninstall='' upgrade='' openssl_path='openssl' - use_pbkdf2='' + kdf='' digest='' salt_method='' salt_method='' @@ -1372,14 +1374,14 @@ transcrypt_main() { digest=${1#*=} ;; -pbkdf2) - use_pbkdf2=1 + kdf=pbkdf2 ;; - --use-pbkdf2) - use_pbkdf2=${2} + --kdf) + kdf=${2} shift ;; - --use-pbkdf2=*) - use_pbkdf2=${1#*=} + --kdf=*) + kdf=${1#*=} ;; -sm | --salt-method) salt_method=$2 @@ -1526,7 +1528,7 @@ transcrypt_main() { # perform function calls to configure transcrypt get_cipher get_digest - get_use_pbkdf2 + get_kdf get_salt_method get_password From 377c61ed939c9bf592c2762641c2ee6dac11ec73 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 14 Jun 2022 13:04:27 -0400 Subject: [PATCH 04/38] Rename salt variable to final_salt to distinguish it --- transcrypt | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/transcrypt b/transcrypt index e40c1e7..5075730 100755 --- a/transcrypt +++ b/transcrypt @@ -80,9 +80,9 @@ _openssl_encrypt() { # to ensure it is always included regardless of the OpenSSL version. #133 ( # Always prepend encrypted ciphertext with "Salted__" prefix and binary salt value - printf "Salted__" && printf "%s" "$salt" | xxd -r -p && + printf "Salted__" && printf "%s" "$final_salt" | xxd -r -p && # Encrypt file to binary ciphertext - ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$salt" "${pbkdf2_args[@]}" -in "$tempfile" | + ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$final_salt" "${pbkdf2_args[@]}" -in "$tempfile" | # Strip "Salted__" prefix and salt value if also added by OpenSSL (version < 3) LC_ALL=C sed -e "s/^\(Salted__.\{8\}\)\(.*\)/\2/" ) | base64 @@ -216,15 +216,15 @@ _load_config_var() { local key=$1 _load_unversioned_config_var "${key}" if [[ "$?" != "0" ]]; then - if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then - _load_versioned_config_var "${key}" - if [[ "$?" != "0" ]]; then - #(echo "loaded ${key} from versioned config") - true - else - (echo "unable to load ${key}") - fi - fi + if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then + _load_versioned_config_var "${key}" + if [[ "$?" != "0" ]]; then + #(echo "loaded ${key} from versioned config") + true + else + (echo "unable to load ${key}") + fi + fi else #(echo "loaded ${key} from unversioned config") true @@ -380,7 +380,7 @@ git_clean() { else _load_vars_for_encryption # NOTE: the openssl standard for salt is 16 hex bytes. - salt=$("$openssl_path" dgst -hmac "${filename}:${extra_salt}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16) + final_salt=$("$openssl_path" dgst -hmac "${filename}:${extra_salt}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16) _openssl_encrypt fi } @@ -1205,16 +1205,16 @@ help() { defaults to md5 --kdf=PBKDF2 - the key-derivation-function to use. Currently can be either - 'pbkdf2' or 'none'. Defaults to none. + the key-derivation-function to use. Currently can be either + 'pbkdf2' or 'none'. Defaults to none. - -pbkdf2 + -pbkdf2 equivalent to passing --kdf2='pbkdf2' -sm, --salt_method=SALT_METHOD - Method used to compute deterministic salt; can be password, random, - or a custom string to be used as the salt. Unless set to password, - the salt is randomized on a rekey. + Method used to compute deterministic salt; can be 'password', 'random', + or a custom string to be used as the salt. Unless set to password, + the salt is randomized on a rekey. --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH From 0b5b25f80ecc93ee39d0828de59ffc2234134500 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 14 Jun 2022 13:24:00 -0400 Subject: [PATCH 05/38] Various fixes Catch validation errors on CLI Remove bad echo Update references to obsolete argument --use-pbdkf2 to --kdf --- contrib/bash/transcrypt | 2 +- contrib/zsh/_transcrypt | 2 +- docs/algorithm.rst | 2 +- example/end_to_end_example.sh | 2 +- example/legacy_upgrade_example.sh | 2 +- tests/test_init.bats | 6 ++-- transcrypt | 46 ++++++++++++------------------- 7 files changed, 25 insertions(+), 37 deletions(-) diff --git a/contrib/bash/transcrypt b/contrib/bash/transcrypt index cee7332..026fc36 100644 --- a/contrib/bash/transcrypt +++ b/contrib/bash/transcrypt @@ -38,7 +38,7 @@ _transcrypt() { COMPREPLY=( $(compgen -W "${ciphers}" -- ${cur}) ) return 0 ;; - --use-pbkdf2) + --kdf) return 0 ;; -sm | --salt-method) diff --git a/contrib/zsh/_transcrypt b/contrib/zsh/_transcrypt index 8f8afa5..97f8939 100644 --- a/contrib/zsh/_transcrypt +++ b/contrib/zsh/_transcrypt @@ -14,7 +14,7 @@ _transcrypt() { '(-md --digest -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify encryption digest]:digest' \ '(-sm --salt-method -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify salt-method]:salt-method' \ '(-cs --config-salt -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify config-salt]:config-salt' \ - '(-pbkdf2 --use-pbkdf2 -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify use-pbkdf2]:use-pbkdf2' \ + '(-pbkdf2 --kdf -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify use-pbkdf2]:use-pbkdf2' \ '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ '(-d --display -p --password -c --cipher -r --rekey -u --uninstall)'{-d,--display}'[display current credentials]' \ diff --git a/docs/algorithm.rst b/docs/algorithm.rst index a56b989..1210afa 100644 --- a/docs/algorithm.rst +++ b/docs/algorithm.rst @@ -148,7 +148,7 @@ in the config transcrypt will set ``pbkdf2_args`` to an empty bash array in the case where pbkdf2 is disabled, and ``-pbkdf2`` otherwise. This allows us to use bash array syntax to express both variants as a single openssl command. -The backwards compatible setting for transcrypt < 3.x is ``--use-pbkdf2=0``. +The backwards compatible setting for transcrypt < 3.x is ``--kdf=0``. See Also: diff --git a/example/end_to_end_example.sh b/example/end_to_end_example.sh index 5ea917e..137826f 100644 --- a/example/end_to_end_example.sh +++ b/example/end_to_end_example.sh @@ -29,7 +29,7 @@ mkdir -p "$DEMO_REPO"/safe # Configure transcrypt with legacy defaults -transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --use-pbkdf2=1 -sm random -y +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=1 -sm random -y git commit -m "Configured transcrypt" echo "Secret contents" > "$DEMO_REPO"/safe/secret_file diff --git a/example/legacy_upgrade_example.sh b/example/legacy_upgrade_example.sh index 5678bfa..cf622c0 100644 --- a/example/legacy_upgrade_example.sh +++ b/example/legacy_upgrade_example.sh @@ -29,7 +29,7 @@ mkdir -p "$DEMO_REPO"/safe # Configure transcrypt with legacy defaults -transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --use-pbkdf2=0 -sm password -y +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=0 -sm password -y echo "Secret contents" > "$DEMO_REPO"/safe/secret_file cat "$DEMO_REPO"/safe/secret_file diff --git a/tests/test_init.bats b/tests/test_init.bats index 3caa324..cc11d3e 100755 --- a/tests/test_init.bats +++ b/tests/test_init.bats @@ -61,7 +61,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --use-pbkdf2 '0' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --kdf '0' -sm 'password'"* ]] } @test "init: show details for -d" { @@ -73,7 +73,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --use-pbkdf2 '0' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --kdf '0' -sm 'password'"* ]] } @test "init: respects core.hooksPath setting" { @@ -90,7 +90,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --use-pbkdf2 '0' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --kdf '0' -sm 'password'"* ]] } @test "init: transcrypt.openssl-path config setting defaults to 'openssl'" { diff --git a/transcrypt b/transcrypt index 5075730..a51a063 100755 --- a/transcrypt +++ b/transcrypt @@ -27,13 +27,10 @@ readonly VERSION='3.0.0-pre' # the default cipher to utilize readonly DEFAULT_CIPHER='aes-256-cbc' -readonly DEFAULT_DIGEST='MD5' +readonly DEFAULT_DIGEST='md5' readonly DEFAULT_KDF='none' readonly DEFAULT_SALT_METHOD='password' -# These are the implemented methods for computing deterministic salt -#readonly VALID_SALT_METHODS="password configured random" - # These are config variables we do not allow to be used in the versioned # configuration readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path" @@ -42,7 +39,7 @@ readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=0} # Proposed option: can remove or disable later -readonly USE_VERSIONED_CONFIG="0" +readonly USE_VERSIONED_CONFIG="1" ##### FUNCTIONS @@ -125,6 +122,7 @@ _validate_variable_str() { if [[ $interactive ]]; then _set_global "$varname" "" echo "$message" + return 1 else die 1 "$message" fi @@ -154,6 +152,7 @@ _get_user_input() { ${validate_fn} fi done + ${validate_fn} || die "Invalid setting" } # Check if the first arg is contained in the space separated second arg @@ -180,7 +179,7 @@ _load_versioned_config_var() { warn "Cannot use ${key} in versioned the transcrypt config" return 1 fi - git config --file "${VERSIONED_TC_CONFIG}" --get "${key}" || true + git config --file "${VERSIONED_TC_CONFIG}" --get "${key}" } # Write a config var to the versioned config @@ -214,21 +213,8 @@ _load_config_var() { # (which is expected to be stored in plaintext and checked into the repo) # Certain values will be blocked from being placed here (like the password) local key=$1 - _load_unversioned_config_var "${key}" - if [[ "$?" != "0" ]]; then - if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then - _load_versioned_config_var "${key}" - if [[ "$?" != "0" ]]; then - #(echo "loaded ${key} from versioned config") - true - else - (echo "unable to load ${key}") - fi - fi - else - #(echo "loaded ${key} from unversioned config") - true - fi + _load_unversioned_config_var "${key}" || \ + ([[ "$USE_VERSIONED_CONFIG" == "1" ]] && _load_versioned_config_var "${key}") } _set_config_var() { @@ -251,6 +237,8 @@ _load_transcrypt_config_vars() { openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) ensure_salt_method + validate_kdf || die "invalid value of kdf in config" + validate_digest || die "invalid value of digest in config" } _load_vars_for_encryption() { @@ -579,8 +567,7 @@ validate_kdf() { } validate_salt_method() { - #_validate_variable_str "salt_method" "$VALID_SALT_METHODS" - _="noop" + true } # ensure we have a digest to hash the salted password @@ -588,7 +575,7 @@ get_digest() { local prompt prompt=$(printf 'Encrypt using which digest? [%s] ' "$DEFAULT_DIGEST") if [[ "$digest" == "" ]]; then - digest=$(_load_versioned_config_var "transcrypt.digest") + digest=$(_load_versioned_config_var "transcrypt.digest" || echo "") # echo "Loaded digest = $digest from local config" fi _get_user_input digest "$DEFAULT_DIGEST" "validate_digest" "$prompt" @@ -599,7 +586,7 @@ get_cipher() { local prompt prompt=$(printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER") if [[ "$cipher" == "" ]]; then - cipher=$(_load_versioned_config_var "transcrypt.cipher") + cipher=$(_load_versioned_config_var "transcrypt.cipher" || echo "") # echo "Loaded cipher = $cipher from local config" fi _get_user_input cipher "$DEFAULT_CIPHER" "validate_cipher" "$prompt" @@ -609,7 +596,7 @@ get_kdf() { local prompt prompt=$(printf 'Which key derivation function? [%s] ' "$DEFAULT_KDF") if [[ "$kdf" == "" ]]; then - kdf=$(_load_versioned_config_var "transcrypt.kdf") + kdf=$(_load_versioned_config_var "transcrypt.kdf" || echo "") # echo "Loaded kdf = $kdf from local config" fi _get_user_input kdf "$DEFAULT_KDF" "validate_kdf" "$prompt" @@ -617,9 +604,9 @@ get_kdf() { get_salt_method() { local prompt - prompt=$(printf 'Compute salt using which method? Can use "password", enter a custom salt, or specify "random". [%s] ' "$DEFAULT_SALT_METHOD") + prompt=$(printf 'Which salt method? [%s] ' "$DEFAULT_SALT_METHOD") if [[ "$salt_method" == "" ]]; then - salt_method=$(_load_versioned_config_var "transcrypt.salt-method") + salt_method=$(_load_versioned_config_var "transcrypt.salt-method" || echo "") # echo "Loaded salt_method = $salt_method from local config" fi _get_user_input salt_method "$DEFAULT_SALT_METHOD" "validate_salt_method" "$prompt" @@ -732,6 +719,7 @@ save_helper_scripts() { # Editable mode is for debugging ln -fs "$current_transcrypt" "${CRYPT_DIR}/transcrypt" else + rm -f "${CRYPT_DIR}/transcrypt" # unlink if it already exist cp -f "$current_transcrypt" "${CRYPT_DIR}/transcrypt" fi # make scripts executable @@ -824,7 +812,7 @@ _display_git_configuration() { # Show the config of the current runtime _display_runtime_configuration() { printf ' DIGEST: %s\n' "$digest" - printf ' USE_PBKDF2: %s\n' "$kdf" + printf ' KDF: %s\n' "$kdf" printf ' SALT_METHOD: %s\n' "$salt_method" if [[ "$salt_method" == "configured" ]]; then printf ' CONFIG_SALT: %s\n' "$salt_method" From b618e8738e96efe4db5f0c6d5581c6d27b0db37b Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 27 Jun 2022 22:53:51 +1000 Subject: [PATCH 06/38] Comply with `shfmt` --- transcrypt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transcrypt b/transcrypt index a51a063..f982b5f 100755 --- a/transcrypt +++ b/transcrypt @@ -122,7 +122,7 @@ _validate_variable_str() { if [[ $interactive ]]; then _set_global "$varname" "" echo "$message" - return 1 + return 1 else die 1 "$message" fi @@ -152,7 +152,7 @@ _get_user_input() { ${validate_fn} fi done - ${validate_fn} || die "Invalid setting" + ${validate_fn} || die "Invalid setting" } # Check if the first arg is contained in the space separated second arg @@ -213,8 +213,8 @@ _load_config_var() { # (which is expected to be stored in plaintext and checked into the repo) # Certain values will be blocked from being placed here (like the password) local key=$1 - _load_unversioned_config_var "${key}" || \ - ([[ "$USE_VERSIONED_CONFIG" == "1" ]] && _load_versioned_config_var "${key}") + _load_unversioned_config_var "${key}" || + ([[ "$USE_VERSIONED_CONFIG" == "1" ]] && _load_versioned_config_var "${key}") } _set_config_var() { @@ -237,8 +237,8 @@ _load_transcrypt_config_vars() { openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) ensure_salt_method - validate_kdf || die "invalid value of kdf in config" - validate_digest || die "invalid value of digest in config" + validate_kdf || die "invalid value of kdf in config" + validate_digest || die "invalid value of digest in config" } _load_vars_for_encryption() { @@ -719,7 +719,7 @@ save_helper_scripts() { # Editable mode is for debugging ln -fs "$current_transcrypt" "${CRYPT_DIR}/transcrypt" else - rm -f "${CRYPT_DIR}/transcrypt" # unlink if it already exist + rm -f "${CRYPT_DIR}/transcrypt" # unlink if it already exist cp -f "$current_transcrypt" "${CRYPT_DIR}/transcrypt" fi # make scripts executable From dbf561349b2f95f472dc62591deda9bb9dc6d37f Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 27 Jun 2022 22:54:07 +1000 Subject: [PATCH 07/38] Disable versioned config so existing BATS tests pass --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index f982b5f..249fb78 100755 --- a/transcrypt +++ b/transcrypt @@ -39,7 +39,7 @@ readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=0} # Proposed option: can remove or disable later -readonly USE_VERSIONED_CONFIG="1" +readonly USE_VERSIONED_CONFIG="0" ##### FUNCTIONS From 1ed6cce09a4c8d046a48aa6fff1bae640fb440cc Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 27 Jun 2022 22:59:18 +1000 Subject: [PATCH 08/38] Update tests for changed default arguments `--kdf` default is now 'none' not '0' `--md` default is now 'md5' not 'MD5' --- tests/test_init.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_init.bats b/tests/test_init.bats index cc11d3e..07eade3 100755 --- a/tests/test_init.bats +++ b/tests/test_init.bats @@ -61,7 +61,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --kdf '0' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -sm 'password'"* ]] } @test "init: show details for -d" { @@ -73,7 +73,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --kdf '0' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -sm 'password'"* ]] } @test "init: respects core.hooksPath setting" { @@ -90,7 +90,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'MD5' --kdf '0' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -sm 'password'"* ]] } @test "init: transcrypt.openssl-path config setting defaults to 'openssl'" { From 3dd51a57dccfab1e5ea8dcd7fb77e466ac892683 Mon Sep 17 00:00:00 2001 From: James Murty Date: Mon, 27 Jun 2022 23:32:45 +1000 Subject: [PATCH 09/38] Potential fix for `unbound variable` in MacOS (tests) - Use `:+` bash parameter expansion to include pbkdf2 argument to openssl only if variable is set - Simplify variable `pbkdf2_arg` from list to string, since the `[@]` referencing doesn't work in all cases for MacOS (at least not for the unit tests): an empty list errors with `unbound variable` --- transcrypt | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/transcrypt b/transcrypt index 249fb78..fcafce2 100755 --- a/transcrypt +++ b/transcrypt @@ -75,21 +75,26 @@ _openssl_encrypt() { # In 3.x openssl disabled output of the salt prefix, which we need for determinism. # We take control over outputting the the prefix 'Salted__' with the salt # to ensure it is always included regardless of the OpenSSL version. #133 - ( - # Always prepend encrypted ciphertext with "Salted__" prefix and binary salt value - printf "Salted__" && printf "%s" "$final_salt" | xxd -r -p && - # Encrypt file to binary ciphertext - ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$final_salt" "${pbkdf2_args[@]}" -in "$tempfile" | - # Strip "Salted__" prefix and salt value if also added by OpenSSL (version < 3) - LC_ALL=C sed -e "s/^\(Salted__.\{8\}\)\(.*\)/\2/" - ) | base64 + openssl_major_version=$($openssl_path version | cut -d' ' -f2 | cut -d'.' -f1) + if [ "$openssl_major_version" -ge "3" ]; then + # Encrypt the file to base64, ensuring it includes the prefix 'Salted__' with the salt. #133 + ( + printf "Salted__" && printf "%s" "$final_salt" | xxd -r -p && + # Encrypt file to binary ciphertext + ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$final_salt" ${pbkdf2_arg:+"$pbkdf2_arg"} -in "$tempfile" + ) | + base64 + else + # Encrypt file to base64 ciphertext + ENC_PASS=$password "$openssl_path" enc -e -a "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$final_salt" ${pbkdf2_arg:+"$pbkdf2_arg"} -in "$tempfile" + fi } _openssl_decrypt() { # Expects that the following variables are set: - # password, openssl_path, cipher, digest, pbkdf2_args + # password, openssl_path, cipher, digest, pbkdf2_arg # This works the same across openssl versions - ENC_PASS=$password "$openssl_path" enc -d "-${cipher}" -md "${digest}" -pass env:ENC_PASS -a "$@" "${pbkdf2_args[@]}" + ENC_PASS=$password "$openssl_path" enc -d "-${cipher}" -md "${digest}" -pass env:ENC_PASS -a ${pbkdf2_arg:+"$pbkdf2_arg"} "$@" } # compatible openssl list command @@ -246,9 +251,7 @@ _load_vars_for_encryption() { _load_transcrypt_config_vars if [[ "$kdf" == "1" ]] || [[ "$kdf" == "pbkdf2" ]]; then - pbkdf2_args=('-pbkdf2') - else - pbkdf2_args=() + pbkdf2_arg='-pbkdf2' fi if [[ "$salt_method" == "password" ]]; then From c38a257aeb25467d83467ef121362683395f023a Mon Sep 17 00:00:00 2001 From: James Murty Date: Fri, 1 Jul 2022 21:28:03 +1000 Subject: [PATCH 10/38] Use OpenSSL for B64 encoding not `base64` which differs between Linux and Mac #140 --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index fcafce2..969e935 100755 --- a/transcrypt +++ b/transcrypt @@ -83,7 +83,7 @@ _openssl_encrypt() { # Encrypt file to binary ciphertext ENC_PASS=$password "$openssl_path" enc -e "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$final_salt" ${pbkdf2_arg:+"$pbkdf2_arg"} -in "$tempfile" ) | - base64 + openssl base64 else # Encrypt file to base64 ciphertext ENC_PASS=$password "$openssl_path" enc -e -a "-${cipher}" -md "${digest}" -pass env:ENC_PASS -S "$final_salt" ${pbkdf2_arg:+"$pbkdf2_arg"} -in "$tempfile" From 3a4f578c3d33a17f3b188e3597621342873e69c4 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 13:43:52 -0400 Subject: [PATCH 11/38] remove machine specific lines --- tests/test_transcrypt.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index 77958ef..764c6da 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -1,4 +1,7 @@ """ +This file provides a Python wrapper around the transcrypt API for the purpose +of testing. + Requirements: pip install ubelt pip install gpg_lite @@ -19,8 +22,6 @@ class Transcrypt(ub.NiceRepr): A Python wrapper around the Transcrypt API Example: - >>> import sys, ubelt - >>> sys.path.append(ubelt.expandpath('~/code/transcrypt/tests')) >>> from test_transcrypt import * # NOQA >>> sandbox = DemoSandbox(verbose=1, dpath='special:cache').setup() >>> config = {'digest': 'sha256', @@ -459,8 +460,6 @@ def test_configuration_grid(): xdoctest -m tests/test_transcrypt.py test_configuration_grid Example: - >>> import sys, ubelt - >>> sys.path.append(ubelt.expandpath('~/code/transcrypt/tests')) >>> from test_transcrypt import * # NOQA >>> self = TestCases() >>> self.setup() @@ -496,6 +495,6 @@ def test_configuration_grid(): if __name__ == '__main__': """ CommandLine: - python ~/code/transcrypt/tests/test_transcrypt.py + python tests/test_transcrypt.py """ test_configuration_grid() From 2262fb74b178cc8c2b0e6d8fe20cc3f476dea81a Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 13:52:01 -0400 Subject: [PATCH 12/38] Rename salt-method to base-salt --- tests/test_transcrypt.py | 38 +++++++++++--------- transcrypt | 75 ++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index 764c6da..0dab255 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -26,7 +26,7 @@ class Transcrypt(ub.NiceRepr): >>> sandbox = DemoSandbox(verbose=1, dpath='special:cache').setup() >>> config = {'digest': 'sha256', >>> 'kdf': 'pbkdf2', - >>> 'salt_method': '665896be121e1a0a4a7b18f01780061'} + >>> 'base_salt': '665896be121e1a0a4a7b18f01780061'} >>> self = Transcrypt(sandbox.repo_dpath, >>> config=config, env=sandbox.env, verbose=1) >>> print(self.version()) @@ -57,7 +57,7 @@ class Transcrypt(ub.NiceRepr): 'password': None, 'digest': 'md5', 'kdf': 'none', - 'salt_method': 'password', + 'base_salt': 'password', } def __init__(self, dpath, config=None, env=None, transcript_exe=None, verbose=0): @@ -89,7 +89,7 @@ def _config_args(self): "-p", self.config['password'], "-md", self.config['digest'], "--kdf", self.config['kdf'], - "-sm", self.config['salt_method'], + "-bs", self.config['base_salt'], ] args = [template.format(**self.config) for template in arg_templates] return args @@ -111,7 +111,7 @@ def login(self): args = self._config_args() command = [str(self.transcript_exe), *args, '-y'] self._cmd(command) - self.config['salt_method'] = self._load_unversioned_config()['salt_method'] + self.config['base_salt'] = self._load_unversioned_config()['base_salt'] def logout(self): """ @@ -127,7 +127,7 @@ def rekey(self, new_config): args = self._config_args() command = [str(self.transcript_exe), '--rekey', *args, '-y'] self._cmd(command) - self.config['salt_method'] = self._load_unversioned_config()['salt_method'] + self.config['base_salt'] = self._load_unversioned_config()['base_salt'] def display(self): """ @@ -202,7 +202,7 @@ def _load_unversioned_config(self): 'cipher': self._cmd('git config --get --local transcrypt.cipher')['out'].strip(), 'digest': self._cmd('git config --get --local transcrypt.digest')['out'].strip(), 'kdf': self._cmd('git config --get --local transcrypt.kdf')['out'].strip(), - 'salt_method': self._cmd('git config --get --local transcrypt.salt-method')['out'].strip(), + 'base_salt': self._cmd('git config --get --local transcrypt.base-salt')['out'].strip(), 'password': self._cmd('git config --get --local transcrypt.password')['out'].strip(), 'openssl_path': self._cmd('git config --get --local transcrypt.openssl-path')['out'].strip(), } @@ -237,6 +237,8 @@ def setup(self): self._setup_gpghome() self._setup_gitrepo() self._setup_contents() + if self.verbose > 2: + self._show_manual_env_setup() return self def _setup_gpghome(self): @@ -262,6 +264,8 @@ def _setup_gpghome(self): ub.cmd('find ' + str(self.gpg_home) + r' -type f -exec chmod 600 {} \;', shell=True, cwd=self.gpg_home) ub.cmd('find ' + str(self.gpg_home) + r' -type d -exec chmod 700 {} \;', shell=True, cwd=self.gpg_home) self.env['GNUPGHOME'] = str(self.gpg_home) + if self.verbose: + pass def _setup_gitrepo(self): if self.verbose: @@ -298,7 +302,7 @@ def _setup_contents(self): self.secret_fpath = self.safe_dpath / 'secret.txt' self.secret_fpath.write_text('secret content') - def _manual_hack_info(self): + def _show_manual_env_setup(self): """ Info on how to get an env to run a failing command manually """ @@ -393,7 +397,7 @@ def test_rekey(self): 'password': '12345', 'digest': 'sha256', 'kdf': 'pbkdf2', - 'salt_method': 'random', + 'base_salt': 'random', } raw_before = self.tc.show_raw(self.sandbox.secret_fpath) self.tc.rekey(new_config) @@ -408,7 +412,7 @@ def test_legacy_defaults(): 'password': 'correct horse battery staple', 'digest': 'md5', 'kdf': 'none', - 'salt_method': 'password', + 'base_salt': 'password', } verbose = 1 self = TestCases(config=config, verbose=verbose) @@ -423,7 +427,7 @@ def test_secure_defaults(): 'password': 'correct horse battery staple', 'digest': 'sha512', 'kdf': 'pbkdf2', - 'salt_method': 'random', + 'base_salt': 'random', } verbose = 1 self = TestCases(config=config, verbose=verbose) @@ -438,19 +442,19 @@ def test_configured_salt_changes_on_rekey(): 'password': 'correct horse battery staple', 'digest': 'sha512', 'kdf': 'pbkdf2', - 'salt_method': 'random', + 'base_salt': 'random', } verbose = 1 self = TestCases(config=config, verbose=verbose) self.setup() before_config = self.tc._load_unversioned_config() - self.tc.rekey({'password': '12345', 'salt_method': ''}) + self.tc.rekey({'password': '12345', 'base_salt': ''}) self.sandbox.git.commit('-am commit rekey') after_config = self.tc._load_unversioned_config() assert before_config['password'] != after_config['password'] assert before_config['cipher'] == after_config['cipher'] assert before_config['kdf'] == after_config['kdf'] - assert before_config['salt_method'] == after_config['salt_method'] + assert before_config['base_salt'] == after_config['base_salt'] assert before_config['openssl_path'] == after_config['openssl_path'] @@ -463,7 +467,7 @@ def test_configuration_grid(): >>> from test_transcrypt import * # NOQA >>> self = TestCases() >>> self.setup() - >>> self.sandbox._manual_hack_info() + >>> self.sandbox._show_manual_env_setup() >>> self.test_round_trip() >>> self.test_export_gpg() """ @@ -473,10 +477,10 @@ def test_configuration_grid(): 'password': ['correct horse battery staple'], 'digest': ['md5', 'sha256'], 'kdf': ['none', 'pbkdf2'], - 'salt_method': ['password', 'random', 'mylittlecustomsalt'], + 'base_salt': ['password', 'random', 'mylittlecustomsalt'], } test_grid = list(ub.named_product(basis)) - verbose = 0 + verbose = 3 dpath = 'special:temp' dpath = 'special:cache' for params in ub.ProgIter(test_grid, desc='test configs', freq=1): @@ -485,7 +489,7 @@ def test_configuration_grid(): self.setup() if 1: # Manual debug - self.sandbox._manual_hack_info() + self.sandbox._show_manual_env_setup() self.test_round_trip() self.test_export_gpg() diff --git a/transcrypt b/transcrypt index 969e935..49a4d89 100755 --- a/transcrypt +++ b/transcrypt @@ -238,10 +238,10 @@ _load_transcrypt_config_vars() { cipher=$(_load_config_var "transcrypt.cipher") || (echo "failed to load transcrypt.cipher" && false) digest=$(_load_config_var "transcrypt.digest") || (echo "failed to load transcrypt.digest" && false) kdf=$(_load_config_var "transcrypt.kdf") || (echo "failed to load transcrypt.kdf" && false) - salt_method=$(_load_config_var "transcrypt.salt-method") || (echo "failed to load transcrypt.salt-method" && false) + base_salt=$(_load_config_var "transcrypt.base-salt") || (echo "failed to load transcrypt.base-salt" && false) openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) - ensure_salt_method + ensure_base_salt validate_kdf || die "invalid value of kdf in config" validate_digest || die "invalid value of digest in config" } @@ -254,12 +254,12 @@ _load_vars_for_encryption() { pbkdf2_arg='-pbkdf2' fi - if [[ "$salt_method" == "password" ]]; then + if [[ "$base_salt" == "password" ]]; then extra_salt=$password - elif [[ "$salt_method" == "" ]]; then + elif [[ "$base_salt" == "" ]]; then die "salt must be specified" else - extra_salt=$salt_method + extra_salt=$base_salt fi if [[ "$extra_salt" == "" ]]; then @@ -569,7 +569,7 @@ validate_kdf() { _validate_variable_str "kdf" "0 1 none pbkdf2" } -validate_salt_method() { +validate_base_salt() { true } @@ -605,16 +605,15 @@ get_kdf() { _get_user_input kdf "$DEFAULT_KDF" "validate_kdf" "$prompt" } -get_salt_method() { +get_base_salt() { local prompt - prompt=$(printf 'Which salt method? [%s] ' "$DEFAULT_SALT_METHOD") - if [[ "$salt_method" == "" ]]; then - salt_method=$(_load_versioned_config_var "transcrypt.salt-method" || echo "") - # echo "Loaded salt_method = $salt_method from local config" + prompt=$(printf 'Which base salt? [%s] ' "$DEFAULT_SALT_METHOD") + if [[ "$base_salt" == "" ]]; then + base_salt=$(_load_versioned_config_var "transcrypt.base-salt" || echo "") + # echo "Loaded base_salt = $base_salt from local config" fi - _get_user_input salt_method "$DEFAULT_SALT_METHOD" "validate_salt_method" "$prompt" - ensure_salt_method - # echo "Got salt_method = $salt_method" + _get_user_input base_salt "$DEFAULT_SALT_METHOD" "validate_base_salt" "$prompt" + ensure_base_salt } # ensure we have a password to encrypt with @@ -642,17 +641,17 @@ get_password() { done } -ensure_salt_method() { +ensure_base_salt() { # Check if randomized salt needs to be written - if [[ "$salt_method" == "random" ]]; then + if [[ "$base_salt" == "random" ]]; then # Replace random with something random. - # If we have not configured the salt_method (or we need to rekey), + # If we have not configured the base_salt (or we need to rekey), # then generate new random salt - salt_method=$(openssl rand -hex 32) + base_salt=$(openssl rand -hex 32) fi - if [[ $rekey ]] && [[ $salt_method != "password" ]]; then + if [[ $rekey ]] && [[ $base_salt != "password" ]]; then # Assume we want a new random salt unless we are explicitly using password - salt_method=$(openssl rand -hex 32) + base_salt=$(openssl rand -hex 32) fi } @@ -767,7 +766,7 @@ save_configuration() { _set_config_var "transcrypt.cipher" "$cipher" _set_config_var "transcrypt.digest" "$digest" _set_config_var "transcrypt.kdf" "$kdf" - _set_config_var "transcrypt.salt-method" "$salt_method" + _set_config_var "transcrypt.base-salt" "$base_salt" _set_unversioned_config_var "transcrypt.openssl-path" "$openssl_path" _set_unversioned_config_var "transcrypt.password" "$password" @@ -816,9 +815,9 @@ _display_git_configuration() { _display_runtime_configuration() { printf ' DIGEST: %s\n' "$digest" printf ' KDF: %s\n' "$kdf" - printf ' SALT_METHOD: %s\n' "$salt_method" - if [[ "$salt_method" == "configured" ]]; then - printf ' CONFIG_SALT: %s\n' "$salt_method" + printf ' SALT_METHOD: %s\n' "$base_salt" + if [[ "$base_salt" == "configured" ]]; then + printf ' CONFIG_SALT: %s\n' "$base_salt" fi printf ' CIPHER: %s\n' "$cipher" printf ' PASSWORD: %s\n\n' "$password" @@ -833,8 +832,8 @@ display_configuration() { _display_git_configuration _display_runtime_configuration printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -sm '%s'\n" \ - "$cipher" "$escaped_password" "$digest" "$kdf" "$salt_method" + printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" } # remove transcrypt-related settings from the repository's git config @@ -1114,8 +1113,8 @@ export_gpg() { local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" #printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" - printf 'password=%s\ncipher=%s\ndigest=%s\nkdf=%s\nsalt_method=%s\n\n' \ - "$password" "$cipher" "$digest" "$kdf" "$salt_method" | + printf 'password=%s\ncipher=%s\ndigest=%s\nkdf=%s\nbase_salt=%s\n\n' \ + "$password" "$cipher" "$digest" "$kdf" "$base_salt" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" printf "The transcrypt configuration has been encrypted and exported to:\n%s/crypt/%s.asc\n" "$GIT_DIR" "$gpg_recipient" } @@ -1151,8 +1150,8 @@ import_gpg() { password=$(printf '%s' "$configuration" | grep '^password' | cut -d'=' -f 2-) digest=$(printf '%s' "$configuration" | grep '^digest' | cut -d'=' -f 2-) kdf=$(printf '%s' "$configuration" | grep '^kdf' | cut -d'=' -f 2-) - salt_method=$(printf '%s' "$configuration" | grep '^salt_method' | cut -d'=' -f 2-) - salt_method=$(printf '%s' "$configuration" | grep '^salt_method' | cut -d'=' -f 2-) + base_salt=$(printf '%s' "$configuration" | grep '^base_salt' | cut -d'=' -f 2-) + base_salt=$(printf '%s' "$configuration" | grep '^base_salt' | cut -d'=' -f 2-) } # print this script's usage message to stderr @@ -1202,7 +1201,7 @@ help() { -pbkdf2 equivalent to passing --kdf2='pbkdf2' - -sm, --salt_method=SALT_METHOD + -bs, --base-salt=SALT_METHOD Method used to compute deterministic salt; can be 'password', 'random', or a custom string to be used as the salt. Unless set to password, the salt is randomized on a rekey. @@ -1314,8 +1313,8 @@ transcrypt_main() { openssl_path='openssl' kdf='' digest='' - salt_method='' - salt_method='' + base_salt='' + base_salt='' # used to bypass certain safety checks requires_existing_config='' @@ -1374,12 +1373,12 @@ transcrypt_main() { --kdf=*) kdf=${1#*=} ;; - -sm | --salt-method) - salt_method=$2 + -bs | --base-salt) + base_salt=$2 shift ;; - --salt-method=*) - salt_method=${1#*=} + --base-salt=*) + base_salt=${1#*=} ;; -p | --password) password=$2 @@ -1520,7 +1519,7 @@ transcrypt_main() { get_cipher get_digest get_kdf - get_salt_method + get_base_salt get_password if [[ $rekey ]] && [[ $interactive ]]; then From b721d870a3ff72265168b39df821e328ebeebe41 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 15:54:01 -0400 Subject: [PATCH 13/38] Rework logic for simpler base-salt UX --- CHANGELOG.md | 2 +- contrib/zsh/_transcrypt | 2 +- tests/test_transcrypt.py | 168 +++++++++++++++++++++++++++++++-------- transcrypt | 137 ++++++++++++++++++++++--------- 4 files changed, 233 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6fbce6..bcfa0bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ The format is based on [Keep a Changelog][1], and this project adheres to - Add support for pbkdf2 - Add support for user specified digest -- Add support for new configured salt method +- Add support for new configured base salt - Add .transcrypt versioned directory - Support for OpenSSL 3.x - Add support for development editable install diff --git a/contrib/zsh/_transcrypt b/contrib/zsh/_transcrypt index 97f8939..9b0a3fa 100644 --- a/contrib/zsh/_transcrypt +++ b/contrib/zsh/_transcrypt @@ -12,7 +12,7 @@ _transcrypt() { '(- 1 *)'{-h,--help}'[view help message]' \ '(-c --cipher -d --display -f --flush-credentials -u --uninstall)'{-c,--cipher=}'[specify encryption cipher]:cipher:->cipher' \ '(-md --digest -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify encryption digest]:digest' \ - '(-sm --salt-method -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify salt-method]:salt-method' \ + '(-bs --base-salt -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify base-salt]:base-salt' \ '(-cs --config-salt -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify config-salt]:config-salt' \ '(-pbkdf2 --kdf -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify use-pbkdf2]:use-pbkdf2' \ '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index 0dab255..470fd2d 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -84,14 +84,19 @@ def _cmd(self, command, shell=False, check=True, verbose=None): shell=shell, check=check) def _config_args(self): - arg_templates = [ - "-c", self.config['cipher'], - "-p", self.config['password'], - "-md", self.config['digest'], - "--kdf", self.config['kdf'], - "-bs", self.config['base_salt'], + flags_and_keys = [ + ('-c', 'cipher'), + ('-p', 'password'), + ('-md', 'digest'), + ('--kdf', 'kdf'), + ('-bs', 'base_salt'), ] - args = [template.format(**self.config) for template in arg_templates] + args = [] + for flag, key in flags_and_keys: + value = self.config[key] + if value is not None: + args.append(flag) + args.append(value) return args def is_configured(self): @@ -144,7 +149,8 @@ def version(self): return self._cmd(f'{self.transcript_exe} --version')['out'].rstrip() def _crypt_dir(self): - info = self._cmd('git config --local transcrypt.crypt-dir', check=0) + info = self._cmd('git config --local transcrypt.crypt-dir', check=0, + verbose=0) if info['err'] == 0: crypt_dpath = ub.Path(info['out'].strip()) else: @@ -198,13 +204,15 @@ def upgrade(self): return self._cmd(f'{self.transcript_exe} --upgrade -y') def _load_unversioned_config(self): + if self.verbose > 0: + print('Loading unversioned config') local_config = { - 'cipher': self._cmd('git config --get --local transcrypt.cipher')['out'].strip(), - 'digest': self._cmd('git config --get --local transcrypt.digest')['out'].strip(), - 'kdf': self._cmd('git config --get --local transcrypt.kdf')['out'].strip(), - 'base_salt': self._cmd('git config --get --local transcrypt.base-salt')['out'].strip(), - 'password': self._cmd('git config --get --local transcrypt.password')['out'].strip(), - 'openssl_path': self._cmd('git config --get --local transcrypt.openssl-path')['out'].strip(), + 'cipher': self._cmd('git config --get --local transcrypt.cipher', verbose=0)['out'].strip(), + 'digest': self._cmd('git config --get --local transcrypt.digest', verbose=0)['out'].strip(), + 'kdf': self._cmd('git config --get --local transcrypt.kdf', verbose=0)['out'].strip(), + 'base_salt': self._cmd('git config --get --local transcrypt.base-salt', verbose=0)['out'].strip(), + 'password': self._cmd('git config --get --local transcrypt.password', verbose=0)['out'].strip(), + 'openssl_path': self._cmd('git config --get --local transcrypt.openssl-path', verbose=0)['out'].strip(), } return local_config @@ -237,7 +245,7 @@ def setup(self): self._setup_gpghome() self._setup_gitrepo() self._setup_contents() - if self.verbose > 2: + if self.verbose > 1: self._show_manual_env_setup() return self @@ -314,6 +322,16 @@ def _show_manual_env_setup(self): class TestCases: """ Unit tests to be applied to different transcrypt configurations + + xdoctest -m tests/test_transcrypt.py TestCases + + Example: + >>> from test_transcrypt import * # NOQA + >>> self = TestCases(verbose=2) + >>> self.setup() + >>> self.sandbox._show_manual_env_setup() + >>> self.test_round_trip() + >>> self.test_export_gpg() """ def __init__(self, config=None, dpath=None, verbose=0): @@ -412,7 +430,7 @@ def test_legacy_defaults(): 'password': 'correct horse battery staple', 'digest': 'md5', 'kdf': 'none', - 'base_salt': 'password', + 'base_salt': '', } verbose = 1 self = TestCases(config=config, verbose=verbose) @@ -437,6 +455,10 @@ def test_secure_defaults(): def test_configured_salt_changes_on_rekey(): + """ + CommandLine: + xdoctest -m tests/test_transcrypt.py test_configured_salt_changes_on_rekey + """ config = { 'cipher': 'aes-256-cbc', 'password': 'correct horse battery staple', @@ -444,32 +466,88 @@ def test_configured_salt_changes_on_rekey(): 'kdf': 'pbkdf2', 'base_salt': 'random', } - verbose = 1 + verbose = 2 self = TestCases(config=config, verbose=verbose) self.setup() before_config = self.tc._load_unversioned_config() self.tc.rekey({'password': '12345', 'base_salt': ''}) self.sandbox.git.commit('-am commit rekey') after_config = self.tc._load_unversioned_config() - assert before_config['password'] != after_config['password'] + assert before_config['password'] != after_config['password'], 'password should have changed!' + assert before_config['base_salt'] != after_config['base_salt'], 'salt should have changed!' assert before_config['cipher'] == after_config['cipher'] assert before_config['kdf'] == after_config['kdf'] - assert before_config['base_salt'] == after_config['base_salt'] assert before_config['openssl_path'] == after_config['openssl_path'] +def test_unspecified_salt_without_kdf(): + """ + In this case the salt should default to the password method + """ + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'sha512', + 'kdf': '', + 'base_salt': None, + } + verbose = 2 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert config1['base_salt'] == 'password' + + +def test_unspecified_salt_with_kdf(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'sha512', + 'kdf': 'pbkdf2', + 'base_salt': None, + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert len(config1['base_salt']) == 64 + + +def test_salt_changes_when_kdf_changes(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'sha512', + 'kdf': '', + 'base_salt': None, + } + verbose = 2 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert config1['base_salt'] == 'password' + # Test rekey, base-salt should still be password + self.tc.rekey({'password': '12345'}) + config2 = self.tc._load_unversioned_config() + assert config2['base_salt'] == 'password' + self.sandbox.git.commit('-am commit rekey') + + # Test rekey with kdf=pbkdf2 base-salt should now randomize + self.tc.rekey({'password': '12345', 'kdf': 'pbkdf2', 'base_salt': None}) + config3 = self.tc._load_unversioned_config() + assert len(config3['base_salt']) == 64, 'should have had new random salt' + self.sandbox.git.commit('-am commit rekey') + + # Test rekey going back to no kdf + self.tc.rekey({'password': '12345', 'kdf': 'none', 'base_salt': None}) + config4 = self.tc._load_unversioned_config() + assert config4['base_salt'] == 'password' + + def test_configuration_grid(): """ CommandLine: xdoctest -m tests/test_transcrypt.py test_configuration_grid - - Example: - >>> from test_transcrypt import * # NOQA - >>> self = TestCases() - >>> self.setup() - >>> self.sandbox._show_manual_env_setup() - >>> self.test_round_trip() - >>> self.test_export_gpg() """ # Test that transcrypt works under a variety of config conditions basis = { @@ -477,28 +555,48 @@ def test_configuration_grid(): 'password': ['correct horse battery staple'], 'digest': ['md5', 'sha256'], 'kdf': ['none', 'pbkdf2'], - 'base_salt': ['password', 'random', 'mylittlecustomsalt'], + 'base_salt': ['password', 'random', 'mylittlecustomsalt', None], } + test_grid = list(ub.named_product(basis)) - verbose = 3 + + def validate_test_grid(params): + if params['kdf'] == 'none' and params['base_salt'] != 'password': + return False + if params['kdf'] != 'none' and params['base_salt'] == 'password': + return False + return True + + # Remove invalid configs + valid_test_grid = list(filter(validate_test_grid, test_grid)) + print('valid_test_grid = {}'.format(ub.repr2(valid_test_grid, sort=0, nl=1))) + + verbose = 2 dpath = 'special:temp' dpath = 'special:cache' - for params in ub.ProgIter(test_grid, desc='test configs', freq=1): + for params in ub.ProgIter(valid_test_grid, desc='test configs', freq=1, + verbose=verbose + 1): + if verbose: + print('\n\n') + print('=================') + print('params = {}'.format(ub.repr2(params, nl=1))) + print('=================') + config = params.copy() self = TestCases(config=config, dpath=dpath, verbose=verbose) self.setup() - if 1: - # Manual debug - self.sandbox._show_manual_env_setup() - self.test_round_trip() self.test_export_gpg() self.test_rekey() - + if verbose: + print('=================') if __name__ == '__main__': """ CommandLine: python tests/test_transcrypt.py + + # Runs everything + pytest tests/test_transcrypt.py -v """ test_configuration_grid() diff --git a/transcrypt b/transcrypt index 49a4d89..1eaa0b4 100755 --- a/transcrypt +++ b/transcrypt @@ -29,14 +29,13 @@ readonly VERSION='3.0.0-pre' readonly DEFAULT_CIPHER='aes-256-cbc' readonly DEFAULT_DIGEST='md5' readonly DEFAULT_KDF='none' -readonly DEFAULT_SALT_METHOD='password' # These are config variables we do not allow to be used in the versioned # configuration readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path" # Set to 1 to enable a development editable installation -readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=0} +readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=1} # Proposed option: can remove or disable later readonly USE_VERSIONED_CONFIG="0" @@ -241,7 +240,6 @@ _load_transcrypt_config_vars() { base_salt=$(_load_config_var "transcrypt.base-salt") || (echo "failed to load transcrypt.base-salt" && false) openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) - ensure_base_salt validate_kdf || die "invalid value of kdf in config" validate_digest || die "invalid value of digest in config" } @@ -570,7 +568,15 @@ validate_kdf() { } validate_base_salt() { - true + if [[ "$kdf" != "none" ]]; then + if [[ "$base_salt" == "" ]]; then + die 1 'The base salt is required when using a kdf, but it is unspecified.' + elif [[ "$base_salt" == "random" ]]; then + die 1 'The base=salt is set to random, but it has not been randomized'. + elif [[ "$base_salt" == "password" ]]; then + die 1 'Using base-salt="password" method with a KDF is insecure. Use "random" instead'. + fi + fi } # ensure we have a digest to hash the salted password @@ -579,7 +585,6 @@ get_digest() { prompt=$(printf 'Encrypt using which digest? [%s] ' "$DEFAULT_DIGEST") if [[ "$digest" == "" ]]; then digest=$(_load_versioned_config_var "transcrypt.digest" || echo "") - # echo "Loaded digest = $digest from local config" fi _get_user_input digest "$DEFAULT_DIGEST" "validate_digest" "$prompt" } @@ -590,7 +595,6 @@ get_cipher() { prompt=$(printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER") if [[ "$cipher" == "" ]]; then cipher=$(_load_versioned_config_var "transcrypt.cipher" || echo "") - # echo "Loaded cipher = $cipher from local config" fi _get_user_input cipher "$DEFAULT_CIPHER" "validate_cipher" "$prompt" } @@ -600,20 +604,91 @@ get_kdf() { prompt=$(printf 'Which key derivation function? [%s] ' "$DEFAULT_KDF") if [[ "$kdf" == "" ]]; then kdf=$(_load_versioned_config_var "transcrypt.kdf" || echo "") - # echo "Loaded kdf = $kdf from local config" fi _get_user_input kdf "$DEFAULT_KDF" "validate_kdf" "$prompt" } get_base_salt() { - local prompt - prompt=$(printf 'Which base salt? [%s] ' "$DEFAULT_SALT_METHOD") - if [[ "$base_salt" == "" ]]; then - base_salt=$(_load_versioned_config_var "transcrypt.base-salt" || echo "") - # echo "Loaded base_salt = $base_salt from local config" - fi - _get_user_input base_salt "$DEFAULT_SALT_METHOD" "validate_base_salt" "$prompt" - ensure_base_salt + __doc__=" + +-----------+-------------+------------------+--------------------+------------------------+ + | Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | + |-----------+-------------+------------------+--------------------+------------------------| + | configure | True | True | True | Use Argv and Overwrite | + | configure | True | True | False | Use Argv and Overwrite | + | configure | True | False | True | Use Config Value | + | configure | True | False | False | Randomize | + | configure | False | True | True | Ignored | + | configure | False | True | False | Ignored | + | configure | False | False | True | Ignored | + | configure | False | False | False | Ignored | + +-----------+-------------+------------------+--------------------+------------------------+ + +----------+-------------+------------------+--------------------+------------------------+ + | Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | + |----------+-------------+------------------+--------------------+------------------------| + | rekey | True | True | True | Use Argv and Overwrite | + | rekey | True | True | False | Use Argv and Overwrite | + | rekey | True | False | True | Randomize | + | rekey | True | False | False | Randomize | + | rekey | False | True | True | Ignored | + | rekey | False | True | False | Ignored | + | rekey | False | False | True | Ignored | + | rekey | False | False | False | Ignored | + +----------+-------------+------------------+--------------------+------------------------+ + + +---------------+-------------+--------------------+------------------+ + | Action | Using_KDF | BaseSaltInConfig | Outcome | + |---------------+-------------+--------------------+------------------| + | helper-script | True | True | Use Config Value | + | helper-script | True | False | Error | + | helper-script | False | True | Ignored | + | helper-script | False | False | Ignored | + +---------------+-------------+--------------------+------------------+ + + # Developer notes to help enumerate cases + import pandas as pd + import ubelt as ub + df = pd.DataFrame(list(ub.named_product({ + 'Action': ['configure', 'rekey', 'helper-script'], + 'Using_KDF': [True, False], + 'BaseSaltInArgv': [True, False], + 'BaseSaltInConfig': [True, False], + }))) + df.loc[~df.Using_KDF, 'Outcome'] = 'Ignored' + df.loc[(df.Action == 'helper-script') & df.BaseSaltInArgv, 'Outcome'] = 'Impossible' + df.loc[df.Using_KDF & (df.Action == 'configure') & df.BaseSaltInArgv, 'Outcome'] = 'Use Argv and Overwrite' + df.loc[df.Using_KDF & (df.Action == 'configure') & ~df.BaseSaltInArgv * ~df.BaseSaltInConfig, 'Outcome'] = 'Randomize' + df.loc[df.Using_KDF & (df.Action == 'configure') & ~df.BaseSaltInArgv * df.BaseSaltInConfig, 'Outcome'] = 'Use Config Value' + df.loc[df.Using_KDF & (df.Action == 'rekey') & df.BaseSaltInArgv, 'Outcome'] = 'Use Argv and Overwrite' + df.loc[df.Using_KDF & (df.Action == 'rekey') & ~df.BaseSaltInArgv, 'Outcome'] = 'Randomize' + df.loc[df.Using_KDF & (df.Action == 'helper-script') & ~df.BaseSaltInArgv & df.BaseSaltInConfig, 'Outcome'] = 'Use Config Value' + df.loc[df.Using_KDF & (df.Action == 'helper-script') & ~df.BaseSaltInArgv & ~df.BaseSaltInConfig, 'Outcome'] = 'Error' + for key, group in df.groupby('Action'): + print(group.to_markdown(index=0, tablefmt='psql')) + " + # When calling `get_base_salt`, the action is always configure or rekey. + if [[ "$kdf" == "none" ]]; then + if [[ "$base_salt" == "" ]]; then + base_salt="password" + fi + else + if [[ "$base_salt" == "" ]]; then + # The user did not specify a base salt. + if [[ $rekey ]]; then + base_salt="random" + else + base_salt=$(_load_versioned_config_var "transcrypt.base-salt" || echo "random") + fi + else + # The user specified something for base-salt + if [[ "$base_salt" == "password" ]]; then + die 1 'Using the "password" method with a KDF is insecure. Use "random" instead'. + fi + fi + # Randomize the base salt if requested. + if [[ "$base_salt" == "random" ]]; then + base_salt=$(openssl rand -hex 32) + fi + fi } # ensure we have a password to encrypt with @@ -641,20 +716,6 @@ get_password() { done } -ensure_base_salt() { - # Check if randomized salt needs to be written - if [[ "$base_salt" == "random" ]]; then - # Replace random with something random. - # If we have not configured the base_salt (or we need to rekey), - # then generate new random salt - base_salt=$(openssl rand -hex 32) - fi - if [[ $rekey ]] && [[ $base_salt != "password" ]]; then - # Assume we want a new random salt unless we are explicitly using password - base_salt=$(openssl rand -hex 32) - fi -} - # confirm the transcrypt configuration confirm_configuration() { local answer= @@ -815,10 +876,7 @@ _display_git_configuration() { _display_runtime_configuration() { printf ' DIGEST: %s\n' "$digest" printf ' KDF: %s\n' "$kdf" - printf ' SALT_METHOD: %s\n' "$base_salt" - if [[ "$base_salt" == "configured" ]]; then - printf ' CONFIG_SALT: %s\n' "$base_salt" - fi + printf ' BASE_SALT: %s\n' "$base_salt" printf ' CIPHER: %s\n' "$cipher" printf ' PASSWORD: %s\n\n' "$password" } @@ -1192,19 +1250,20 @@ help() { -md, --digest=DIGEST the digest used to hash the salted password; - defaults to md5 + defaults to md5. It is strongly recommended to use + a stronger hash (e.g. sha256) if possible. --kdf=PBKDF2 the key-derivation-function to use. Currently can be either - 'pbkdf2' or 'none'. Defaults to none. + 'pbkdf2' or 'none'. Defaults to none. It is strongly + recommended to use a kdf (e.g. pbkdf2) if possible. -pbkdf2 equivalent to passing --kdf2='pbkdf2' - -bs, --base-salt=SALT_METHOD - Method used to compute deterministic salt; can be 'password', 'random', - or a custom string to be used as the salt. Unless set to password, - the salt is randomized on a rekey. + -bs, --base-salt=BASE_SALT + if specified, and a KDF is used, this overrides the text + used as the basis for salt-generation. --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH From 047fff258c61223a27254bf04c705152e8c681fd Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 16:01:37 -0400 Subject: [PATCH 14/38] Ensure digest is converted to lowercase --- example/end_to_end_example.sh | 2 +- transcrypt | 1 + transcrypt_bashlib.sh | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/example/end_to_end_example.sh b/example/end_to_end_example.sh index 137826f..60f3f57 100644 --- a/example/end_to_end_example.sh +++ b/example/end_to_end_example.sh @@ -29,7 +29,7 @@ mkdir -p "$DEMO_REPO"/safe # Configure transcrypt with legacy defaults -transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=1 -sm random -y +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=1 -y git commit -m "Configured transcrypt" echo "Secret contents" > "$DEMO_REPO"/safe/secret_file diff --git a/transcrypt b/transcrypt index 1eaa0b4..fc127ad 100755 --- a/transcrypt +++ b/transcrypt @@ -586,6 +586,7 @@ get_digest() { if [[ "$digest" == "" ]]; then digest=$(_load_versioned_config_var "transcrypt.digest" || echo "") fi + digest=$(echo "${digest}" | tr '[:upper:]' '[:lower:]') _get_user_input digest "$DEFAULT_DIGEST" "validate_digest" "$prompt" } diff --git a/transcrypt_bashlib.sh b/transcrypt_bashlib.sh index 0d588ac..64df202 100644 --- a/transcrypt_bashlib.sh +++ b/transcrypt_bashlib.sh @@ -449,3 +449,10 @@ _benchmark_methods(){ #bash_array_repr "${available[@]}" } + +_lowercase(){ + __doc__=' + _lowercase "FOO" + ' + echo "$1" | tr '[:upper:]' '[:lower:]' +} From b85cfe1f5f69637feb31de70004eab84618505ca Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 16:08:28 -0400 Subject: [PATCH 15/38] Fix issue where KDF was not remembered through rekey --- tests/test_transcrypt.py | 25 +++++++++++++++++++++++++ transcrypt | 8 ++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index 470fd2d..97e4761 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -544,6 +544,31 @@ def test_salt_changes_when_kdf_changes(): assert config4['base_salt'] == 'password' +def test_kdf_setting_preserved_on_rekey(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'md5', + 'kdf': 'pbkdf2', + 'base_salt': None, + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert len(config1['base_salt']) == 64 + + # Explicitly don't pass kdf or base salt. + # Transcrypt should reuse the existing kdf setting (but the salt should + # change) + self.tc.rekey({'password': '12345', 'kdf': None, 'base_salt': None, 'digest': 'SHA512'}) + config2 = self.tc._load_unversioned_config() + assert config2['kdf'] == 'pbkdf2' + assert config1['base_salt'] != config2['base_salt'] + assert len(config1['base_salt']) == 64 + assert len(config2['base_salt']) == 64 + + def test_configuration_grid(): """ CommandLine: diff --git a/transcrypt b/transcrypt index fc127ad..91c0883 100755 --- a/transcrypt +++ b/transcrypt @@ -584,7 +584,7 @@ get_digest() { local prompt prompt=$(printf 'Encrypt using which digest? [%s] ' "$DEFAULT_DIGEST") if [[ "$digest" == "" ]]; then - digest=$(_load_versioned_config_var "transcrypt.digest" || echo "") + digest=$(_load_config_var "transcrypt.digest" || echo "") fi digest=$(echo "${digest}" | tr '[:upper:]' '[:lower:]') _get_user_input digest "$DEFAULT_DIGEST" "validate_digest" "$prompt" @@ -595,7 +595,7 @@ get_cipher() { local prompt prompt=$(printf 'Encrypt using which cipher? [%s] ' "$DEFAULT_CIPHER") if [[ "$cipher" == "" ]]; then - cipher=$(_load_versioned_config_var "transcrypt.cipher" || echo "") + cipher=$(_load_config_var "transcrypt.cipher" || echo "") fi _get_user_input cipher "$DEFAULT_CIPHER" "validate_cipher" "$prompt" } @@ -604,7 +604,7 @@ get_kdf() { local prompt prompt=$(printf 'Which key derivation function? [%s] ' "$DEFAULT_KDF") if [[ "$kdf" == "" ]]; then - kdf=$(_load_versioned_config_var "transcrypt.kdf" || echo "") + kdf=$(_load_config_var "transcrypt.kdf" || echo "") fi _get_user_input kdf "$DEFAULT_KDF" "validate_kdf" "$prompt" } @@ -677,7 +677,7 @@ get_base_salt() { if [[ $rekey ]]; then base_salt="random" else - base_salt=$(_load_versioned_config_var "transcrypt.base-salt" || echo "random") + base_salt=$(_load_config_var "transcrypt.base-salt" || echo "random") fi else # The user specified something for base-salt From 31257bf74818f5e975b20f9ed6660c29b904a125 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 16:39:01 -0400 Subject: [PATCH 16/38] Give user the ability to disable the versioned config --- example/end_to_end_example.sh | 4 +- example/legacy_upgrade_example.sh | 2 +- transcrypt | 80 +++++++++++++++++++++++++------ 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/example/end_to_end_example.sh b/example/end_to_end_example.sh index 60f3f57..2d54834 100644 --- a/example/end_to_end_example.sh +++ b/example/end_to_end_example.sh @@ -28,7 +28,7 @@ git commit -m "add attributes" mkdir -p "$DEMO_REPO"/safe -# Configure transcrypt with legacy defaults +# Configure transcrypt with a KDF, but an old hash function transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=1 -y git commit -m "Configured transcrypt" @@ -40,7 +40,7 @@ git commit -m "add secret with config1" transcrypt -s safe/secret_file -# Rekey with more secure settings +# Rekey with a new more secure hash function transcrypt --rekey -c aes-256-cbc -p 'foobar' -md SHA256 -y git commit -am "changed crypto settings" diff --git a/example/legacy_upgrade_example.sh b/example/legacy_upgrade_example.sh index cf622c0..66eb0f3 100644 --- a/example/legacy_upgrade_example.sh +++ b/example/legacy_upgrade_example.sh @@ -29,7 +29,7 @@ mkdir -p "$DEMO_REPO"/safe # Configure transcrypt with legacy defaults -transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=0 -sm password -y +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=0 -y echo "Secret contents" > "$DEMO_REPO"/safe/secret_file cat "$DEMO_REPO"/safe/secret_file diff --git a/transcrypt b/transcrypt index 91c0883..3355ced 100755 --- a/transcrypt +++ b/transcrypt @@ -37,9 +37,6 @@ readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path # Set to 1 to enable a development editable installation readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=1} -# Proposed option: can remove or disable later -readonly USE_VERSIONED_CONFIG="0" - ##### FUNCTIONS # print a canonicalized absolute pathname @@ -218,7 +215,7 @@ _load_config_var() { # Certain values will be blocked from being placed here (like the password) local key=$1 _load_unversioned_config_var "${key}" || - ([[ "$USE_VERSIONED_CONFIG" == "1" ]] && _load_versioned_config_var "${key}") + ([[ "$use_versioned_config" == "1" ]] && _load_versioned_config_var "${key}") } _set_config_var() { @@ -226,7 +223,7 @@ _set_config_var() { local key=$1 local val=$2 _set_unversioned_config_var "$key" "$val" - if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then + if [[ "$use_versioned_config" == "1" ]]; then _set_versioned_config_var "$key" "$val" fi } @@ -234,6 +231,7 @@ _set_config_var() { # shellcheck disable=SC2155 _load_transcrypt_config_vars() { # Populate bash vars with our config + use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config") || (echo "failed to load transcrypt.use-versioned-config" && false) cipher=$(_load_config_var "transcrypt.cipher") || (echo "failed to load transcrypt.cipher" && false) digest=$(_load_config_var "transcrypt.digest") || (echo "failed to load transcrypt.digest" && false) kdf=$(_load_config_var "transcrypt.kdf") || (echo "failed to load transcrypt.kdf" && false) @@ -379,7 +377,6 @@ git_smudge() { # the file contents are passed via stdin tempfile=$(mktemp 2>/dev/null || mktemp -t tmp) trap 'rm -f "$tempfile"' EXIT - #_load_transcrypt_config_vars _load_vars_for_encryption tee "$tempfile" | _openssl_decrypt 2>/dev/null || cat "$tempfile" } @@ -554,17 +551,26 @@ run_safety_checks() { validate_cipher() { local valid_ciphers valid_ciphers=$(_openssl_list cipher-commands) + cipher=$(echo "${cipher}" | tr '[:upper:]' '[:lower:]') _validate_variable_str "cipher" "$valid_ciphers" } validate_digest() { local valid_digests valid_digests=$(_openssl_list digest-commands) + digest=$(echo "${digest}" | tr '[:upper:]' '[:lower:]') _validate_variable_str "digest" "$valid_digests" } validate_kdf() { - _validate_variable_str "kdf" "0 1 none pbkdf2" + # Normalize kdf + kdf=$(echo "${kdf}" | tr '[:upper:]' '[:lower:]') + if [[ "$kdf" == "0" ]]; then + kdf="none" + elif [[ "$kdf" == "1" ]]; then + kdf="pbkdf2" + fi + _validate_variable_str "kdf" "none pbkdf2" } validate_base_salt() { @@ -579,6 +585,29 @@ validate_base_salt() { fi } + +get_use_versioned_config() { + __doc__=" + If the user did not specify to use the versioned config or not, then: + + 1. Check if there is a setting in the unversioned config. + 2. If not, check if the versioned config exists. + 3. If not, then defer further checking until we know our setting of base-salt. + " + echo "in predetermine use_versioned_config = $use_versioned_config" + if [[ "$use_versioned_config" == "" ]]; then + use_versioned_config=$(_load_unversioned_config "transcrypt.use-versioned-config" || echo "") + fi + if [[ "$use_versioned_config" == "" ]]; then + if [ -f "${VERSIONED_TC_CONFIG}" ]; then + use_versioned_config="1" + else + use_versioned_config="defer" # defer to after base-salt setting is read + fi + fi + echo "predetermine use_versioned_config = $use_versioned_config" +} + # ensure we have a digest to hash the salted password get_digest() { local prompt @@ -586,7 +615,6 @@ get_digest() { if [[ "$digest" == "" ]]; then digest=$(_load_config_var "transcrypt.digest" || echo "") fi - digest=$(echo "${digest}" | tr '[:upper:]' '[:lower:]') _get_user_input digest "$DEFAULT_DIGEST" "validate_digest" "$prompt" } @@ -666,11 +694,17 @@ get_base_salt() { for key, group in df.groupby('Action'): print(group.to_markdown(index=0, tablefmt='psql')) " + echo "before base salt use_versioned_config = $use_versioned_config" + echo "kdf = $kdf" # When calling `get_base_salt`, the action is always configure or rekey. if [[ "$kdf" == "none" ]]; then if [[ "$base_salt" == "" ]]; then base_salt="password" fi + if [[ "$use_versioned_config" == "defer" ]]; then + # Handle deferred use_version_config + use_versioned_config="0" + fi else if [[ "$base_salt" == "" ]]; then # The user did not specify a base salt. @@ -689,7 +723,12 @@ get_base_salt() { if [[ "$base_salt" == "random" ]]; then base_salt=$(openssl rand -hex 32) fi + if [[ "$use_versioned_config" == "defer" ]]; then + # Handle deferred use_version_config + use_versioned_config="1" + fi fi + echo "after base salt use_versioned_config = $use_versioned_config" } # ensure we have a password to encrypt with @@ -824,6 +863,7 @@ save_configuration() { save_helper_hooks # write the encryption info + _set_unversioned_config_var "transcrypt.use-versioned-config" "$use_versioned_config" _set_config_var "transcrypt.version" "$VERSION" _set_config_var "transcrypt.cipher" "$cipher" _set_config_var "transcrypt.digest" "$digest" @@ -832,7 +872,7 @@ save_configuration() { _set_unversioned_config_var "transcrypt.openssl-path" "$openssl_path" _set_unversioned_config_var "transcrypt.password" "$password" - if [[ "$USE_VERSIONED_CONFIG" == "1" ]]; then + if [[ "$use_versioned_config" == "1" ]]; then if [ -f $RELATIVE_VERSIONED_TC_CONFIG ]; then if ! git ls-files --error-unmatch "$RELATIVE_VERSIONED_TC_CONFIG" >/dev/null 2>&1; then git add "${RELATIVE_VERSIONED_TC_CONFIG}" @@ -1164,14 +1204,9 @@ export_gpg() { _load_transcrypt_config_vars - #local current_cipher - #current_cipher=$(git config --get --local transcrypt.cipher) - #local current_password - #current_password=$(git config --get --local transcrypt.password) mkdir -p "${CRYPT_DIR}" local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" - #printf 'password=%s\ncipher=%s\n' "$current_password" "$current_cipher" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" printf 'password=%s\ncipher=%s\ndigest=%s\nkdf=%s\nbase_salt=%s\n\n' \ "$password" "$cipher" "$digest" "$kdf" "$base_salt" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" @@ -1266,6 +1301,11 @@ help() { if specified, and a KDF is used, this overrides the text used as the basis for salt-generation. + -vc, --versioned-config=<0|1> + Force transcrypt to use or not use the versioned config. + By default the versioned config is only used when using a + KDF. + --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH @@ -1374,7 +1414,7 @@ transcrypt_main() { kdf='' digest='' base_salt='' - base_salt='' + use_versioned_config='' # used to bypass certain safety checks requires_existing_config='' @@ -1447,6 +1487,13 @@ transcrypt_main() { --password=*) password=${1#*=} ;; + -vc | --versioned-config) + use_versioned_config=$2 + shift + ;; + --versioned-config=*) + use_versioned_config=${1#*=} + ;; --set-openssl-path=*) openssl_path=${1#*=} # Immediately apply config setting @@ -1541,6 +1588,9 @@ transcrypt_main() { # always run our safety checks run_safety_checks + # determine if we are using the versioned config or not + get_use_versioned_config + # regular expression used to test user input readonly YES_REGEX='^[Yy]$' From cd9e0000631535543ba8ee52b62b9847f42a456d Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 17:11:09 -0400 Subject: [PATCH 17/38] cleanup --- .github/workflows/run-bats-core-tests.yml | 2 +- tests/test_transcrypt.py | 73 +++- transcrypt | 109 ++--- transcrypt_bashlib.sh | 458 ---------------------- 4 files changed, 112 insertions(+), 530 deletions(-) delete mode 100644 transcrypt_bashlib.sh diff --git a/.github/workflows/run-bats-core-tests.yml b/.github/workflows/run-bats-core-tests.yml index ccc4b76..bd326a3 100644 --- a/.github/workflows/run-bats-core-tests.yml +++ b/.github/workflows/run-bats-core-tests.yml @@ -20,7 +20,7 @@ jobs: - name: Run shellcheck and shfmt uses: luizm/action-sh-checker@master with: - sh_checker_exclude: "tests example transcrypt_bashlib.sh" + sh_checker_exclude: "tests example" sh_checker_comment: true test: diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index 97e4761..41607b7 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -58,6 +58,7 @@ class Transcrypt(ub.NiceRepr): 'digest': 'md5', 'kdf': 'none', 'base_salt': 'password', + 'use_versioned_config': None, } def __init__(self, dpath, config=None, env=None, transcript_exe=None, verbose=0): @@ -90,10 +91,11 @@ def _config_args(self): ('-md', 'digest'), ('--kdf', 'kdf'), ('-bs', 'base_salt'), + ('-vc', 'use_versioned_config'), ] args = [] for flag, key in flags_and_keys: - value = self.config[key] + value = self.config.get(key, None) if value is not None: args.append(flag) args.append(value) @@ -207,6 +209,7 @@ def _load_unversioned_config(self): if self.verbose > 0: print('Loading unversioned config') local_config = { + 'use_versioned_config': self._cmd('git config --get --local transcrypt.use-versioned-config', verbose=0)['out'].strip(), 'cipher': self._cmd('git config --get --local transcrypt.cipher', verbose=0)['out'].strip(), 'digest': self._cmd('git config --get --local transcrypt.digest', verbose=0)['out'].strip(), 'kdf': self._cmd('git config --get --local transcrypt.kdf', verbose=0)['out'].strip(), @@ -513,6 +516,72 @@ def test_unspecified_salt_with_kdf(): assert len(config1['base_salt']) == 64 +def test_legacy_settings_dont_use_the_versioned_config(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'md5', + 'kdf': 'none', + 'base_salt': None, + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert not (self.sandbox.dpath / '.transcrypt').exists() + assert config1['use_versioned_config'] == '0' + + +def test_pbkdf_does_use_the_versioned_config(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'md5', + 'kdf': 'pbkdf2', + 'base_salt': None, + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert config1['use_versioned_config'] == '1' + assert not (self.sandbox.dpath / '.transcrypt').exists() + + +def test_force_use_versioned_config_1(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'md5', + 'kdf': 'none', + 'base_salt': None, + 'use_versioned_config': '1', + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert config1['use_versioned_config'] == '1' + assert not (self.sandbox.dpath / '.transcrypt').exists() + + +def test_force_use_versioned_config_0(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'md5', + 'kdf': 'pbkdf2', + 'base_salt': None, + 'use_versioned_config': '0', + } + verbose = 1 + self = TestCases(config=config, verbose=verbose) + self.setup() + config1 = self.tc._load_unversioned_config() + assert not (self.sandbox.dpath / '.transcrypt').exists() + assert config1['use_versioned_config'] == '0' + + def test_salt_changes_when_kdf_changes(): config = { 'cipher': 'aes-256-cbc', @@ -561,7 +630,7 @@ def test_kdf_setting_preserved_on_rekey(): # Explicitly don't pass kdf or base salt. # Transcrypt should reuse the existing kdf setting (but the salt should # change) - self.tc.rekey({'password': '12345', 'kdf': None, 'base_salt': None, 'digest': 'SHA512'}) + self.tc.rekey({'kdf': None, 'base_salt': None, 'digest': 'SHA512'}) config2 = self.tc._load_unversioned_config() assert config2['kdf'] == 'pbkdf2' assert config1['base_salt'] != config2['base_salt'] diff --git a/transcrypt b/transcrypt index 3355ced..e2af0ce 100755 --- a/transcrypt +++ b/transcrypt @@ -35,6 +35,7 @@ readonly DEFAULT_KDF='none' readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path" # Set to 1 to enable a development editable installation +# This will symlink transcrypt into the .git folder instead of copying it readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=1} ##### FUNCTIONS @@ -69,7 +70,7 @@ realpath() { _openssl_encrypt() { # In 3.x openssl disabled output of the salt prefix, which we need for determinism. - # We take control over outputting the the prefix 'Salted__' with the salt + # We take control over outputting the prefix 'Salted__' with the salt # to ensure it is always included regardless of the OpenSSL version. #133 openssl_major_version=$($openssl_path version | cut -d' ' -f2 | cut -d'.' -f1) if [ "$openssl_major_version" -ge "3" ]; then @@ -238,6 +239,16 @@ _load_transcrypt_config_vars() { base_salt=$(_load_config_var "transcrypt.base-salt") || (echo "failed to load transcrypt.base-salt" && false) openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) + + #+---------------+-------------+--------------------+------------------+ + #| Action | Using_KDF | BaseSaltInConfig | Outcome | + #|---------------+-------------+--------------------+------------------| + #| helper-script | True | True | Use Config Value | + #| helper-script | True | False | Error | + #| helper-script | False | True | Ignored | + #| helper-script | False | False | Ignored | + #+---------------+-------------+--------------------+------------------+ + validate_kdf || die "invalid value of kdf in config" validate_digest || die "invalid value of digest in config" } @@ -587,14 +598,10 @@ validate_base_salt() { get_use_versioned_config() { - __doc__=" - If the user did not specify to use the versioned config or not, then: - - 1. Check if there is a setting in the unversioned config. - 2. If not, check if the versioned config exists. - 3. If not, then defer further checking until we know our setting of base-salt. - " - echo "in predetermine use_versioned_config = $use_versioned_config" + #If the user did not specify to use the versioned config or not, then: + # 1. Check if there is a setting in the unversioned config. + # 2. If not, check if the versioned config exists. + # 3. If not, then defer further checking until we know our setting of base-salt. if [[ "$use_versioned_config" == "" ]]; then use_versioned_config=$(_load_unversioned_config "transcrypt.use-versioned-config" || echo "") fi @@ -605,7 +612,6 @@ get_use_versioned_config() { use_versioned_config="defer" # defer to after base-salt setting is read fi fi - echo "predetermine use_versioned_config = $use_versioned_config" } # ensure we have a digest to hash the salted password @@ -638,64 +644,30 @@ get_kdf() { } get_base_salt() { - __doc__=" - +-----------+-------------+------------------+--------------------+------------------------+ - | Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | - |-----------+-------------+------------------+--------------------+------------------------| - | configure | True | True | True | Use Argv and Overwrite | - | configure | True | True | False | Use Argv and Overwrite | - | configure | True | False | True | Use Config Value | - | configure | True | False | False | Randomize | - | configure | False | True | True | Ignored | - | configure | False | True | False | Ignored | - | configure | False | False | True | Ignored | - | configure | False | False | False | Ignored | - +-----------+-------------+------------------+--------------------+------------------------+ - +----------+-------------+------------------+--------------------+------------------------+ - | Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | - |----------+-------------+------------------+--------------------+------------------------| - | rekey | True | True | True | Use Argv and Overwrite | - | rekey | True | True | False | Use Argv and Overwrite | - | rekey | True | False | True | Randomize | - | rekey | True | False | False | Randomize | - | rekey | False | True | True | Ignored | - | rekey | False | True | False | Ignored | - | rekey | False | False | True | Ignored | - | rekey | False | False | False | Ignored | - +----------+-------------+------------------+--------------------+------------------------+ - - +---------------+-------------+--------------------+------------------+ - | Action | Using_KDF | BaseSaltInConfig | Outcome | - |---------------+-------------+--------------------+------------------| - | helper-script | True | True | Use Config Value | - | helper-script | True | False | Error | - | helper-script | False | True | Ignored | - | helper-script | False | False | Ignored | - +---------------+-------------+--------------------+------------------+ - - # Developer notes to help enumerate cases - import pandas as pd - import ubelt as ub - df = pd.DataFrame(list(ub.named_product({ - 'Action': ['configure', 'rekey', 'helper-script'], - 'Using_KDF': [True, False], - 'BaseSaltInArgv': [True, False], - 'BaseSaltInConfig': [True, False], - }))) - df.loc[~df.Using_KDF, 'Outcome'] = 'Ignored' - df.loc[(df.Action == 'helper-script') & df.BaseSaltInArgv, 'Outcome'] = 'Impossible' - df.loc[df.Using_KDF & (df.Action == 'configure') & df.BaseSaltInArgv, 'Outcome'] = 'Use Argv and Overwrite' - df.loc[df.Using_KDF & (df.Action == 'configure') & ~df.BaseSaltInArgv * ~df.BaseSaltInConfig, 'Outcome'] = 'Randomize' - df.loc[df.Using_KDF & (df.Action == 'configure') & ~df.BaseSaltInArgv * df.BaseSaltInConfig, 'Outcome'] = 'Use Config Value' - df.loc[df.Using_KDF & (df.Action == 'rekey') & df.BaseSaltInArgv, 'Outcome'] = 'Use Argv and Overwrite' - df.loc[df.Using_KDF & (df.Action == 'rekey') & ~df.BaseSaltInArgv, 'Outcome'] = 'Randomize' - df.loc[df.Using_KDF & (df.Action == 'helper-script') & ~df.BaseSaltInArgv & df.BaseSaltInConfig, 'Outcome'] = 'Use Config Value' - df.loc[df.Using_KDF & (df.Action == 'helper-script') & ~df.BaseSaltInArgv & ~df.BaseSaltInConfig, 'Outcome'] = 'Error' - for key, group in df.groupby('Action'): - print(group.to_markdown(index=0, tablefmt='psql')) - " - echo "before base salt use_versioned_config = $use_versioned_config" - echo "kdf = $kdf" + #+-----------+-------------+------------------+--------------------+------------------------+ + #| Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | + #|-----------+-------------+------------------+--------------------+------------------------| + #| configure | True | True | True | Use Argv and Overwrite | + #| configure | True | True | False | Use Argv and Overwrite | + #| configure | True | False | True | Use Config Value | + #| configure | True | False | False | Randomize | + #| configure | False | True | True | Ignored | + #| configure | False | True | False | Ignored | + #| configure | False | False | True | Ignored | + #| configure | False | False | False | Ignored | + #+-----------+-------------+------------------+--------------------+------------------------+ + #+----------+-------------+------------------+--------------------+------------------------+ + #| Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | + #|----------+-------------+------------------+--------------------+------------------------| + #| rekey | True | True | True | Use Argv and Overwrite | + #| rekey | True | True | False | Use Argv and Overwrite | + #| rekey | True | False | True | Randomize | + #| rekey | True | False | False | Randomize | + #| rekey | False | True | True | Ignored | + #| rekey | False | True | False | Ignored | + #| rekey | False | False | True | Ignored | + #| rekey | False | False | False | Ignored | + #+----------+-------------+------------------+--------------------+------------------------+ # When calling `get_base_salt`, the action is always configure or rekey. if [[ "$kdf" == "none" ]]; then if [[ "$base_salt" == "" ]]; then @@ -728,7 +700,6 @@ get_base_salt() { use_versioned_config="1" fi fi - echo "after base salt use_versioned_config = $use_versioned_config" } # ensure we have a password to encrypt with diff --git a/transcrypt_bashlib.sh b/transcrypt_bashlib.sh deleted file mode 100644 index 64df202..0000000 --- a/transcrypt_bashlib.sh +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env bash -__doc__=' -This contains the standalone heredoc versions of transcrypt library functions. -These are not used in the main executable itself. Instead they are ported from -here to there and stripped of extranious information. - -This makes it easier to unit test the individual bash components of the system -while still providing a fast and reasonably optimized runtime. -' - -# print a message to stderr -warn() { - local fmt="$1" - shift - # shellcheck disable=SC2059 - printf "transcrypt: $fmt\n" "$@" >&2 -} - -# print a message to stderr and exit with either -# the given status or that of the most recent command -die() { - local st="$?" - if [[ "$1" != *[^0-9]* ]]; then - st="$1" - shift - fi - warn "$@" - exit "$st" -} - -# print a canonicalized absolute pathname -realpath() { - local path=$1 - - # make path absolute - local abspath=$path - if [[ -n ${abspath##/*} ]]; then - abspath=$(pwd -P)/$abspath - fi - - # canonicalize path - local dirname= - if [[ -d $abspath ]]; then - dirname=$(cd "$abspath" && pwd -P) - abspath=$dirname - elif [[ -e $abspath ]]; then - dirname=$(cd "${abspath%/*}/" 2>/dev/null && pwd -P) - abspath=$dirname/${abspath##*/} - fi - - if [[ -d $dirname && -e $abspath ]]; then - printf '%s\n' "$abspath" - else - printf 'invalid path: %s\n' "$path" >&2 - exit 1 - fi -} - -joinby(){ - __doc__=' - A function that works similar to a Python join - - Args: - SEP: the separator - *ARR: elements of the strings to join - - Usage: - source $HOME/local/init/utils.sh - ARR=("foo" "bar" "baz") - RESULT=$(joinby / "${ARR[@]}") - echo "RESULT = $RESULT" - - RESULT = foo/bar/baz - - References: - https://stackoverflow.com/questions/1527049/how-can-i-join-elements-of-an-array-in-bash - ' - _handle_help "$@" || return 0 - local d=${1-} f=${2-} - if shift 2; then - printf %s "$f" "${@/#/$d}" - fi -} - -# shellcheck disable=SC2154 -_openssl_encrypt() -{ - __doc__=' - Example: - source ~/code/transcrypt/transcrypt_bashlib.sh - pbkdf2_args=("-pbkdf2") - salt=deadbeafbad00000 - digest=sha256 - password=12345 - openssl_path=openssl - cipher=aes-256-cbc - tempfile=$(mktemp) - echo "secret" > $tempfile - _openssl_encrypt - ' - # Exepcts that the following variables are set: - # password, openssl_path, cipher, digest, salt, pbkdf2_args, tempfile - - # Test the openssl version - openssl_major_version=$($openssl_path version | cut -d' ' -f2 | cut -d'.' -f1) - if [ "$openssl_major_version" -ge "3" ]; then - # OpenSSL 3.x - # In 3.x openssl disabled output of the salt prefix, which we need for determinism. - # To reenable the prefix we emit the raw prefix bytes, encrypt in raw bytes, and then - # send that entire stream to be base64 encoded - (printf "Salted__" && printf "%s" "$salt" | xxd -r -p && \ - ENC_PASS=$password "$openssl_path" enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS -e -S "$salt" "${pbkdf2_args[@]}" -in "$tempfile" - ) | base64 - else - # OpenSSL 1.x - ENC_PASS=$password "$openssl_path" enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS -e -a -S "$salt" "${pbkdf2_args[@]}" -in "$tempfile" - fi -} - -# shellcheck disable=SC2154 -_openssl_decrypt() -{ - __doc__=' - Example: - source ~/code/transcrypt/transcrypt_bashlib.sh - pbkdf2_args=("-pbkdf2") - digest=sha256 - password=12345 - openssl_path=openssl - cipher=aes-256-cbc - echo "U2FsdGVkX1/erb6vutAAADPXEjWJ3l4MEpSGTj5qC/w=" | _openssl_decrypt - tempfile=$(mktemp) - echo "U2FsdGVkX1/erb6vutAAADPXEjWJ3l4MEpSGTj5qC/w=" > $tempfile - _openssl_decrypt -in $tempfile - ' - # Exepcts that the following variables are set: - # password, openssl_path, cipher, digest, pbkdf2_args - # This works the same across openssl versions - ENC_PASS=$password "$openssl_path" enc "-${cipher}" -md "${digest}" -pass env:ENC_PASS "${pbkdf2_args[@]}" -d -a "$@" -} - -_openssl_list(){ - # Args: the openssl commands to list - __doc__=' - source ~/code/transcrypt/bash_helpers.sh - arg=digest-commands - _openssl_list digest-commands - _openssl_list cipher-commands - ' - openssl_path=openssl - arg=$1 - if "${openssl_path} list-$arg" &>/dev/null; then - # OpenSSL < v1.1.0 - "${openssl_path}" "list-$arg" - else - # OpenSSL >= v1.1.0 - "${openssl_path}" "list" "-$arg" - fi -} - - -_is_contained_str(){ - __doc__=' - Args: - arg : the query to check if it is contained in the values - values : a string of space separated values - - Example: - source ~/code/transcrypt/bash_helpers.sh - # Demo using raw call - (_is_contained_str "foo" "foo bar baz" && echo "contained") || echo "missing" - (_is_contained_str "bar" "foo bar baz" && echo "contained") || echo "missing" - (_is_contained_str "baz" "foo bar baz" && echo "contained") || echo "missing" - (_is_contained_str "biz" "foo bar baz" && echo "contained") || echo "missing" - # Demo using variables - arg="bar" - values="foo bar baz" - (_is_contained_str "$arg" "$values" && echo "contained") || echo "missing" - ' - arg=$1 - values=$2 - echo "$values" | tr -s ' ' '\n' | grep -Fx "$arg" &>/dev/null -} - -_is_contained_arr(){ - __doc__=' - Check if the first value is contained the rest of the values - - Args: - arg : the query to check if it is contained in the values - *values : the rest of the arguments are individual elements in the values - - Example: - source ~/code/transcrypt/bash_helpers.sh - # Demo using raw call - (_is_contained_arr "bar" "foo" "bar" "baz" && echo "contained") || echo "missing" - (_is_contained_arr "biz" "foo" "bar" "baz" && echo "contained") || echo "missing" - # Demo using variables - values=("foo" "bar" "baz") - arg="bar" - (_is_contained_arr "$arg" "${values[@]}" && echo "contained") || echo "missing" - arg="biz" - (_is_contained_arr "$arg" "${values[@]}" && echo "contained") || echo "missing" - ' - # The first argument must be equal to one of the subsequent arguments - local arg=$1 - shift - local arr=("$@") - for val in "${arr[@]}"; - do - if [[ "${arg}" == "${val}" ]]; then - return 0 - fi - done - return 1 -} - -_set_global(){ - # sets a bash global variable by name - key=$1 - val=$2 - printf -v "$key" '%s' "$val" -} - -_validate_variable_arr(){ - __doc__=' - Example: - source bash_helpers.sh - foo="bar" - valid_values=("bar" "biz") - _validate_variable "foo" "${valid_values[@]}" - interactive=1 - _validate_variable "blaz" "${valid_values[@]}" - ' - local varname=$1 - local valid_values=$2 - local varval=${!varname} - if ! _is_contained_arr "$varval" "${valid_values[@]}"; then - local valid_values_str - valid_values_str=$(joinby ', ' "${valid_values[@]}") - message=$(printf "%s is %s, but must be one of: %s" "$varname" "$varval" "$valid_values_str") - if [[ $interactive ]]; then - _set_global "$varname" "" - echo "$message" - else - die 1 "$message" - fi - fi -} - - -_validate_variable_str(){ - __doc__=' - Checks if the target variable is in the set of valid values. - If it is not, it unsets the target variable, then if not in interactive - mode it calls die. - - Args: - varname: name of variable to validate - valid_values: space separated string of valid values - - Example: - source bash_helpers.sh - valid_values="bar biz" - foo="bar" - _validate_variable_str "foo" "$valid_values" - interactive=1 - blaz=fds - _validate_variable_str "blaz" "$valid_values" - ' - local varname=$1 - local valid_values=$2 - local varval=${!varname} - if ! _is_contained_str "$varval" "$valid_values"; then - message=$(printf '%s is `%s`, but must be one of: %s' "$varname" "$varval" "$valid_values") - if [[ $interactive ]]; then - _set_global "$varname" "" - echo "$message" - else - die 1 "$message" - fi - fi -} - -_get_user_input2() { - __doc__=' - Helper to prompt the user, store a response, and validate the result - Args: - varname : name of the bash variable to populate - default : the default value to use if the user provides no answer - valid_values: space separated string of valid values - prompt : string to present to the user - - Example: - source ~/code/transcrypt/bash_helpers.sh - interactive=1 - myvar= - echo "myvar = <$myvar>" - _get_user_input2 "myvar" "a" "a b c" "choose one" - ' - local varname=$1 - local default=$2 - local valid_values=$3 - local prompt=$4 - - while [[ ! ${!varname} ]]; do - local answer= - if [[ $interactive ]]; then - printf '%s > ' "$prompt" - read -r answer - fi - # use the default value if the user gave no answer; otherwise call the - # validate function, which should set the varname to empty if it is - # invalid and the user should continue, otherwise it should die. - if [[ ! $answer ]]; then - _set_global "$varname" "$default" - else - _set_global "$varname" "$answer" - _validate_variable_str "$varname" "$valid_values" - fi - done -} - - -# Helper to prompt the user, store a response, and validate the result -_get_user_input() { - local varname=$1 - local default=$2 - local validate_fn=$3 - local prompt=$4 - - while [[ ! ${!varname} ]]; do - local answer= - if [[ $interactive ]]; then - printf '%s' "$prompt" - read -r answer - fi - # use the default value if the user gave no answer; otherwise call the - # validate function, which should set the varname to empty if it is - # invalid and the user should continue, otherwise it should die. - if [[ ! $answer ]]; then - _set_global "$varname" "$default" - else - _set_global "$varname" "$answer" - ${validate_fn} - fi - done -} - - -# Load a config var from the versioned config -# shellcheck disable=SC2155 -_load_versioned_config_var(){ - # the current git repository's top-level directory - if [ -z "${VERSIONED_TC_CONFIG+x}" ]; then - readonly REPO=$(git rev-parse --show-toplevel 2>/dev/null) - # This is the place where transcrypt can store state that will be - # "versioned" (i.e. checked into the repo) - readonly VERSIONED_TC_DIR="${REPO}/.transcrypt" - readonly VERSIONED_TC_CONFIG="${VERSIONED_TC_DIR}/config" - fi - local key=$1 - # Test for blocked variables that should not go into a plaintext config file - if _is_contained_str "${key}" "${VERSIONED_CONFIG_BLOCKLIST}"; then - warn "Cannot use ${key} in versioned the transcrypt config" - return 1 - fi - # https://unix.stackexchange.com/questions/175648/use-config-file-for-my-shell-script - git config --file "${VERSIONED_TC_CONFIG}" --get "${key}" || true -} - -# Write a config var to the versioned config -_set_versioned_config_var(){ - local key=$1 - local val=$2 - # Test for blocked variables that should not go into a plaintext config file - if _is_contained_str "${key}" "${VERSIONED_CONFIG_BLOCKLIST}"; then - warn "Cannot use ${key} in versioned the transcrypt config" - return 1 - fi - mkdir -p "${VERSIONED_TC_DIR}" - git config --file "${VERSIONED_TC_CONFIG}" "${key}" "${val}" -} - -# -_load_config_var(){ - # First try loading from the local checkout-independent .git/config file - # If that doesn't work, then look in the .transcrypt/config file - # (which is expected to be stored in plaintext and checked into the repo) - # Certain values will be blocked from being placed here (like the password) - local key=$1 - git config --get --local "${key}" - if [[ "$?" != "0" ]]; then - _load_versioned_config_var "${key}" - fi -} - -# shellcheck disable=SC2155 -_load_transcrypt_config_vars(){ - # Populate bash vars with our config - cipher=$(git config --get --local transcrypt.cipher) - digest=$(git config --get --local transcrypt.digest) - use_pbkdf2=$(git config --get --local transcrypt.use-pbkdf2) - salt_method=$(git config --get --local transcrypt.salt-method) - - password=$(git config --get --local transcrypt.password) - openssl_path=$(git config --get --local transcrypt.openssl-path) - - if [[ "$salt_method" == "configured" ]]; then - config_salt=$(_load_versioned_config_var "transcrypt.config-salt") - else - config_salt="" - fi -} - -_load_vars_for_encryption(){ - # Helper to populate variables needed to call openssl encryption - _load_transcrypt_config_vars - - if [[ "$use_pbkdf2" == "1" ]]; then - pbkdf2_args=('-pbkdf2') - else - pbkdf2_args=() - fi - - if [[ "$salt_method" == "password" ]]; then - extra_salt=$password - elif [[ "$salt_method" == "configured" ]]; then - extra_salt=$config_salt - else - die "unknown salt method" - fi - - if [[ "$extra_salt" == "" ]]; then - die "Extra salt is not set" - fi -} - -_benchmark_methods(){ - # Development helepr to determine which way of checking if we have a available digest / cipher is fastest - arg="sha512" - source ~/code/transcrypt/bash_helpers.sh - time (openssl list -digest-commands | tr -s ' ' '\n' | grep -Fx "$arg") - echo $? - time _is_contained_str "$arg" "$(openssl list -digest-commands)" - echo $? - time (readarray -t available <<< "$(openssl list -digest-commands | tr -s ' ' '\n')" && _is_contained_arr "$arg" "${available[@]}") - echo $? - #bash_array_repr "${available[@]}" -} - - -_lowercase(){ - __doc__=' - _lowercase "FOO" - ' - echo "$1" | tr '[:upper:]' '[:lower:]' -} From 0f7b457834fec17f89f213f26fd68cace887d814 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 17:39:48 -0400 Subject: [PATCH 18/38] update algo writeup --- CHANGELOG.md | 2 +- docs/algorithm.rst | 121 +++++++++++---------------------------- tests/test_transcrypt.py | 3 +- transcrypt | 18 ++++-- 4 files changed, 49 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfa0bf..b2f120c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog][1], and this project adheres to - Add support for pbkdf2 - Add support for user specified digest - Add support for new configured base salt -- Add .transcrypt versioned directory +- Add support for an optional .transcrypt versioned directory - Support for OpenSSL 3.x - Add support for development editable install diff --git a/docs/algorithm.rst b/docs/algorithm.rst index 1210afa..276a040 100644 --- a/docs/algorithm.rst +++ b/docs/algorithm.rst @@ -52,7 +52,7 @@ Given a sensitive file specified by ``filename`` digest, key derivation function, salt, and password), openssl is called to encrypt the plaintext, and the base64 ciphertext is emitted and passed to git. -The following is the openssl invocation used in encryption +The following is (similar to) the openssl invocation used in encryption .. code:: bash @@ -81,7 +81,7 @@ in the user's working branch via the ``filter.crypt.smudge`` filter. decryption fails the ciphertext itself is emitted via stdout. -The following invocation is used for decryption +The following invocation is (similar to) the command used for decryption .. code:: bash @@ -92,23 +92,20 @@ The following invocation is used for decryption Configuration -============= +------------- Loading the configuration is a critical subroutine in the core transcrypt components. -In the proposed transcrypt 3.x implementation, the following *bash* variables -are required for encryption and decryption: +In the proposed transcrypt 3.x implementation, the following variables are +required for encryption and decryption: * ``cipher`` * ``password`` * ``digest`` -* ``pbkdf2_args`` - +* ``kdf`` +* ``base_salt`` -And additionally, encryption needs the variable: - -* ``salt`` Cipher, Password, and Digest ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -125,7 +122,7 @@ in plaintext. Note, that before transcrypt 3.x only cipher and password were configurable. Legacy behavior of transcrypt is described by assuming digest is MD5. -The other two variables ``pbkdf2_args`` and ``salt`` are less straight forward. +The other two variables ``kdf`` and ``base_salt`` are less straight forward. PBKDF2 @@ -141,24 +138,23 @@ OpenSSL enables ``pbkdf2`` if the ``-pbkdf2`` flag is specified. To coerce this into a key-value configuration scheme we use the git configuration variable -* ``transcrypt.use-pbkdf2`` +* ``transcrypt.kdf`` -Which can be set to 0 or 1. At configuration load time, depending on the value -in the config transcrypt will set ``pbkdf2_args`` to an empty bash array in the -case where pbkdf2 is disabled, and ``-pbkdf2`` otherwise. This allows us to use -bash array syntax to express both variants as a single openssl command. +Which can be set to "none" or "pbkdf2", which will enable the ``-pbkdf2`` +openssl flag in the latter case. -The backwards compatible setting for transcrypt < 3.x is ``--kdf=0``. +The backwards compatible setting for transcrypt < 3.x is ``--kdf=none``. See Also: PKCS5#5.2 (RFC-2898) https://datatracker.ietf.org/doc/html/rfc2898#section-5.2 -Salt -~~~~ +Base Salt +~~~~~~~~~ -Lastly, there is ``salt``, which the least straightforward of these options. +Lastly, there is ``base_salt``, which influences how we determine the final +salt for the encryption process. Ideally, when using openssl, a unique and random salt is generated **each time** the file is encrypted. This prevents an attacker from executing a @@ -173,7 +169,7 @@ changed file every time the "clean" command was executed. Transcrypt therefore defines two strategies to generate a deterministic salt: 1. The "password" salt method. -2. The "configured" salt method. +2. The "random" salt method. The first method is equivalent to the existing process in transcrypt 2.x. The second method is a new more secure variant, but will rely on a new @@ -190,32 +186,24 @@ generated for each file via the following invocation: This salt is based on the name of the file, its sha256 hash, and something called "extra-salt", which is determined by the user's choice of -``transcrypt.salt-method``. +``transcrypt.kdf`` and ``transcrypt.base-salt``. -In the case where ``transcrypt.salt-method=password``, the "extra-salt" is set -to the user's plaintext password. This exactly mimics the behavior of -transcrypt 2.x and is used as the default to provide backwards compatibility. +In the case where ``transcrypt.kdf=none``, the "extra-salt" is set +to the user's plaintext password and ``transcrypt.base-salt`` is ignored. This +exactly mimics the behavior of transcrypt 2.x and is used as the default to +provide backwards compatibility. However, as discussed in `#55 _`, this introduces a security weakness that weakens the extra security provided the use of -``-pbkdf2``. Thus transcrypt 3.x introduces a new "configured" method. - -In the case where ``transcrypt.salt-method=configured``, the implementation -will check if a special configuration variable ``transcrypt.config-salt`` is -set, and if not, it will set it to a random 32 character hex string, and check -the choice of that value into the repo. Then the value of -``transcrypt.config-salt`` will be used as "extra-salt". The value of -``transcrypt.config-salt`` is randomized every time the user changes their -password. We note that this method this method does provide less entropy than -randomly choosing the salt on each encryption cycle, but we are unaware of -any security concerns that arise from this method. +``-pbkdf2``. Thus, transcrypt 3.x introduces a new "random" method. -Note: this method could be further improved by generated a randomized -config-salt for each file that is modified when the file itself is modified. -Such a scheme should exactly match the entropy of the openssl default -randomized salt method. However, due to the added implementation complexity -and unclear security benefits we defer that to future work. +In the case where ``transcrypt.kdf=pbkdf2``, transcrypt will store a randomized +(32 character hex string) or custom user-specified string in +``transcrypt.base-salt``. This value is rerandomized on a rekey. We note that +this method this method does provide less entropy than randomly choosing the +salt on each encryption cycle, but we are unaware of +any security concerns that arise from this method. See Also: @@ -267,52 +255,9 @@ Second, transcrypt 3.x adds 4 new parameters that a user will need to configure. By storing these parameters in the repo itself it will ease the burden of decrypting a fresh clone of a repo. -Using this versioned config for everything but ``transcrypt.config-salt`` is -completely optional (and using ``transcrypt.config-salt`` is not needed if -``transcrypt.salt-method=password``, although that is not recommended). Thus -the user can still choose to keep the chosen cipher, digest, and use of pbkdf2 -a secret if they desire (although we will remind the reader that +We also introduce an option to disable the versioned config by specifying +``--versioned-config=0`` on the command line. Thus the user can still choose to +keep the chosen cipher, digest, use of pbkdf2, and base-salt a secret if they +desire (although we will remind the reader that `security by obscurity _` should never be relied on). - -NOTE: Currently, as of 2022-05-09, the current implementation of transcrypt 3.x -does not implement the ability for ``.transcrypt/config`` to store any config -variable other than ``transcrypt.config-salt``. We will wait for this proposal -to be reviewed because the design of the priority in which configuration -variables are stored is is currently an open question in the mind of the -author. However, proposed example *behavior* is as follows: - -Case Study and Open Questions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Given: A fresh clone of a repo without a ``.transcrypt/config`` file. - -The user invokes ``transcrypt`` and is prompted for all 6 configuration variables. - -These are stored to the primary ``.git/config`` file, except for -``transcrypt.config-salt``, which --- if the salt method is "configured" --- is -always stored in ``.git/transcrypt`` and checked into the repo. The user is notified -that transcrypt used ``git add`` to stage this file, and instructs the user to commit -the file (transcrypt never invokes the ``git commit`` command). - -Proposal: The user is additionally prompted if they want to add the -non-sensitive configuration to the versioned config. This prompt can be skipped -by specifying ``--versioned=1`` or ``--versioned=0``. In the unversioned case, -the process proceeded as-is, otherwise the non-sensitive configuration is written -to ``.transcrypt/config`` **instead of** being written to ``.git/config``. - -Open Question: When non-sensitive configuration variables are written, should they be: - -1. Written only to ``.transcrypt/config`` and not ``.git/config``? -2. Written to both ``.transcrypt/config`` and ``.git/config``? -3. Written only to ``.transcrypt/config`` and ensured they are removed from ``.git/config``? - -Because all of these configuration files are plain-text and editable we have to -consider the precedence of config settings when loading. The current proposal -is to always look at ``.git/config`` first and then fallback to -``.transcrypt/config``. - -Open Question: When we read a variable from ``.git/config`` and it disagrees -with ``.transcrypt/config`` do we "fix" ``.transcrypt/config``, warn, or ignore -it. My current proposal is to ignore it and rely on documented precedence -rules. diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index 41607b7..ac2b52c 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -650,6 +650,7 @@ def test_configuration_grid(): 'digest': ['md5', 'sha256'], 'kdf': ['none', 'pbkdf2'], 'base_salt': ['password', 'random', 'mylittlecustomsalt', None], + 'use_versioned_config': ['0', '1', None], } test_grid = list(ub.named_product(basis)) @@ -691,6 +692,6 @@ def validate_test_grid(params): python tests/test_transcrypt.py # Runs everything - pytest tests/test_transcrypt.py -v + pytest tests/test_transcrypt.py -v -s """ test_configuration_grid() diff --git a/transcrypt b/transcrypt index e2af0ce..30ec99c 100755 --- a/transcrypt +++ b/transcrypt @@ -902,8 +902,16 @@ display_configuration() { _display_git_configuration _display_runtime_configuration printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s'\n" \ - "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" + + if [[ "$use_versioned_config" == "0" ]] && [[ "$kdf" != "none" ]] ; then + # The user is forcing the versioned config off, give them an invocation + # that respects that + printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s' -vc '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" "$use_versioned_config" + else + printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" + fi } # remove transcrypt-related settings from the repository's git config @@ -1178,8 +1186,8 @@ export_gpg() { mkdir -p "${CRYPT_DIR}" local gpg_encrypt_cmd="gpg --batch --recipient $gpg_recipient --trust-model always --yes --armor --quiet --encrypt -" - printf 'password=%s\ncipher=%s\ndigest=%s\nkdf=%s\nbase_salt=%s\n\n' \ - "$password" "$cipher" "$digest" "$kdf" "$base_salt" | + printf 'password=%s\ncipher=%s\ndigest=%s\nkdf=%s\nbase_salt=%s\nuse_versioned_config=%s\n\n' \ + "$password" "$cipher" "$digest" "$kdf" "$base_salt" "$use_versioned_config" | $gpg_encrypt_cmd >"${CRYPT_DIR}/${gpg_recipient}.asc" printf "The transcrypt configuration has been encrypted and exported to:\n%s/crypt/%s.asc\n" "$GIT_DIR" "$gpg_recipient" } @@ -1216,7 +1224,7 @@ import_gpg() { digest=$(printf '%s' "$configuration" | grep '^digest' | cut -d'=' -f 2-) kdf=$(printf '%s' "$configuration" | grep '^kdf' | cut -d'=' -f 2-) base_salt=$(printf '%s' "$configuration" | grep '^base_salt' | cut -d'=' -f 2-) - base_salt=$(printf '%s' "$configuration" | grep '^base_salt' | cut -d'=' -f 2-) + use_versioned_config=$(printf '%s' "$configuration" | grep '^use_versioned_config' | cut -d'=' -f 2-) } # print this script's usage message to stderr From 11c499984ecc0cf8fa62fb87c536b62b6a4176ca Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 17:44:41 -0400 Subject: [PATCH 19/38] Fixup docs and indent --- contrib/bash/transcrypt | 5 +- contrib/zsh/_transcrypt | 5 +- man/transcrypt.1 | 65 +++--------- transcrypt | 222 ++++++++++++++++++++-------------------- 4 files changed, 131 insertions(+), 166 deletions(-) diff --git a/contrib/bash/transcrypt b/contrib/bash/transcrypt index 026fc36..2e02fbd 100644 --- a/contrib/bash/transcrypt +++ b/contrib/bash/transcrypt @@ -41,10 +41,7 @@ _transcrypt() { --kdf) return 0 ;; - -sm | --salt-method) - return 0 - ;; - -cs | --config-salt) + -bs | --base-salt) return 0 ;; -s | --show-raw) diff --git a/contrib/zsh/_transcrypt b/contrib/zsh/_transcrypt index 9b0a3fa..e4aa28e 100644 --- a/contrib/zsh/_transcrypt +++ b/contrib/zsh/_transcrypt @@ -12,9 +12,8 @@ _transcrypt() { '(- 1 *)'{-h,--help}'[view help message]' \ '(-c --cipher -d --display -f --flush-credentials -u --uninstall)'{-c,--cipher=}'[specify encryption cipher]:cipher:->cipher' \ '(-md --digest -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify encryption digest]:digest' \ - '(-bs --base-salt -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify base-salt]:base-salt' \ - '(-cs --config-salt -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify config-salt]:config-salt' \ - '(-pbkdf2 --kdf -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify use-pbkdf2]:use-pbkdf2' \ + '(-bs --base-salt -d --display -f --flush-credentials -u --uninstall)'{-bs,--base-salt=}'[specify base-salt]:base-salt' \ + '(-pbkdf2 --kdf -d --display -f --flush-credentials -u --uninstall)'{-pbkdf2,--kdf=}'[specify kdf]:kdf' \ '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ '(-d --display -p --password -c --cipher -r --rekey -u --uninstall)'{-d,--display}'[display current credentials]' \ diff --git a/man/transcrypt.1 b/man/transcrypt.1 index 7b1d979..0ab304a 100644 --- a/man/transcrypt.1 +++ b/man/transcrypt.1 @@ -1,118 +1,87 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "TRANSCRYPT" "1" "August 2016" "" "" -. +.\" generated with Ronn-NG/v0.9.1 +.\" http://github.com/apjanke/ronn-ng/tree/0.9.1 +.TH "TRANSCRYPT" "1" "June 2022" "" .SH "NAME" \fBtranscrypt\fR \- transparently encrypt files within a git repository -. .SH "SYNOPSIS" -\fBtranscrypt\fR [\fIoptions\fR\.\.\.] -. +\fBtranscrypt\fR [\fIoptions\fR\|\.\|\.\|\.] .SH "DESCRIPTION" -transcrypt will configure a Git repository to support the transparent encryption/decryption of files by utilizing OpenSSL\'s symmetric cipher routines and Git\'s built\-in clean/smudge filters\. It will also add a Git alias "ls\-crypt" to list all transparently encrypted files within the repository\. -. +transcrypt will configure a Git repository to support the transparent encryption/decryption of files by utilizing OpenSSL's symmetric cipher routines and Git's built\-in clean/smudge filters\. It will also add a Git alias "ls\-crypt" to list all transparently encrypted files within the repository\. .P The transcrypt source code and full documentation may be downloaded from \fIhttps://github\.com/elasticdog/transcrypt\fR\. -. .SH "OPTIONS" -. .TP \fB\-c\fR, \fB\-\-cipher\fR=\fIcipher\fR the symmetric cipher to utilize for encryption; defaults to aes\-256\-cbc -. .TP \fB\-p\fR, \fB\-\-password\fR=\fIpassword\fR the password to derive the key from; defaults to 30 random base64 characters -. +.TP +\fB\-\-set\-openssl\-path\fR=\fIpath_to_openssl\fR +use OpenSSL at this path; defaults to 'openssl' in $PATH .TP \fB\-y\fR, \fB\-\-yes\fR assume yes and accept defaults for non\-specified options -. .TP \fB\-d\fR, \fB\-\-display\fR -display the current repository\'s cipher and password -. +display the current repository's cipher and password .TP \fB\-r\fR, \fB\-\-rekey\fR re\-encrypt all encrypted files using new credentials -. .TP \fB\-f\fR, \fB\-\-flush\-credentials\fR remove the locally cached encryption credentials and re\-encrypt any files that had been previously decrypted -. .TP \fB\-F\fR, \fB\-\-force\fR ignore whether the git directory is clean, proceed with the possibility that uncommitted changes are overwritten -. .TP \fB\-u\fR, \fB\-\-uninstall\fR remove all transcrypt configuration from the repository and leave files in the current working copy decrypted -. +.TP +\fB\-\-upgrade\fR +apply the latest transcrypt scripts in the repository without changing your configuration settings .TP \fB\-l\fR, \fB\-\-list\fR list all of the transparently encrypted files in the repository, relative to the top\-level directory -. .TP \fB\-s\fR, \fB\-\-show\-raw\fR=\fIfile\fR show the raw file as stored in the git commit object; use this to check if files are encrypted as expected -. .TP \fB\-e\fR, \fB\-\-export\-gpg\fR=\fIrecipient\fR -export the repository\'s cipher and password to a file encrypted for a gpg recipient -. +export the repository's cipher and password to a file encrypted for a gpg recipient .TP \fB\-i\fR, \fB\-\-import\-gpg\fR=\fIfile\fR import the password and cipher from a gpg encrypted file -. .TP \fB\-v\fR, \fB\-\-version\fR print the version information -. .TP \fB\-h\fR, \fB\-\-help\fR view this help message -. .SH "EXAMPLES" To initialize a Git repository to support transparent encryption, just change into the repo and run the transcrypt script\. transcrypt will prompt you interactively for all required information if the corresponding option flags were not given\. -. .IP "" 4 -. .nf - $ cd / $ transcrypt -. .fi -. .IP "" 0 -. .P -Once a repository has been configured with transcrypt, you can transparently encrypt files by applying the "crypt" filter and diff to a pattern in the top\-level \fI\.gitattributes\fR config\. If that pattern matches a file in your repository, the file will be transparently encrypted once you stage and commit it: -. +Once a repository has been configured with transcrypt, you can transparently encrypt files by applying the "crypt" filter, diff and merge to a pattern in the top\-level \fI\.gitattributes\fR config\. If that pattern matches a file in your repository, the file will be transparently encrypted once you stage and commit it: .IP "" 4 -. .nf - -$ echo \'sensitive_file filter=crypt diff=crypt\' >> \.gitattributes +$ echo 'sensitive_file filter=crypt diff=crypt merge=crypt' >> \.gitattributes $ git add \.gitattributes sensitive_file -$ git commit \-m \'Add encrypted version of a sensitive file\' -. +$ git commit \-m 'Add encrypted version of a sensitive file' .fi -. .IP "" 0 -. .P See the gitattributes(5) man page for more information\. -. .P -If you have just cloned a repository containing files that are encrypted, you\'ll want to configure transcrypt with the same cipher and password as the origin repository\. Once transcrypt has stored the matching credentials, it will force a checkout of any existing encrypted files in order to decrypt them\. -. +If you have just cloned a repository containing files that are encrypted, you'll want to configure transcrypt with the same cipher and password as the origin repository\. Once transcrypt has stored the matching credentials, it will force a checkout of any existing encrypted files in order to decrypt them\. .P If the origin repository has just rekeyed, all clones should flush their transcrypt credentials, fetch and merge the new encrypted files via Git, and then re\-configure transcrypt with the new credentials\. -. .SH "AUTHOR" Aaron Bull Schaefer -. .SH "SEE ALSO" enc(1), gitattributes(5) diff --git a/transcrypt b/transcrypt index 30ec99c..7267514 100755 --- a/transcrypt +++ b/transcrypt @@ -240,14 +240,14 @@ _load_transcrypt_config_vars() { openssl_path=$(_load_config_var "transcrypt.openssl-path") || (echo "failed to load transcrypt.openssl-path" && false) password=$(_load_unversioned_config_var transcrypt.password) || (echo "failed to load transcrypt.password" && false) - #+---------------+-------------+--------------------+------------------+ - #| Action | Using_KDF | BaseSaltInConfig | Outcome | - #|---------------+-------------+--------------------+------------------| - #| helper-script | True | True | Use Config Value | - #| helper-script | True | False | Error | - #| helper-script | False | True | Ignored | - #| helper-script | False | False | Ignored | - #+---------------+-------------+--------------------+------------------+ + #+---------------+-------------+--------------------+------------------+ + #| Action | Using_KDF | BaseSaltInConfig | Outcome | + #|---------------+-------------+--------------------+------------------| + #| helper-script | True | True | Use Config Value | + #| helper-script | True | False | Error | + #| helper-script | False | True | Ignored | + #| helper-script | False | False | Ignored | + #+---------------+-------------+--------------------+------------------+ validate_kdf || die "invalid value of kdf in config" validate_digest || die "invalid value of digest in config" @@ -562,56 +562,56 @@ run_safety_checks() { validate_cipher() { local valid_ciphers valid_ciphers=$(_openssl_list cipher-commands) - cipher=$(echo "${cipher}" | tr '[:upper:]' '[:lower:]') + cipher=$(echo "${cipher}" | tr '[:upper:]' '[:lower:]') _validate_variable_str "cipher" "$valid_ciphers" } validate_digest() { local valid_digests valid_digests=$(_openssl_list digest-commands) - digest=$(echo "${digest}" | tr '[:upper:]' '[:lower:]') + digest=$(echo "${digest}" | tr '[:upper:]' '[:lower:]') _validate_variable_str "digest" "$valid_digests" } validate_kdf() { - # Normalize kdf - kdf=$(echo "${kdf}" | tr '[:upper:]' '[:lower:]') - if [[ "$kdf" == "0" ]]; then - kdf="none" - elif [[ "$kdf" == "1" ]]; then - kdf="pbkdf2" - fi + # Normalize kdf + kdf=$(echo "${kdf}" | tr '[:upper:]' '[:lower:]') + if [[ "$kdf" == "0" ]]; then + kdf="none" + elif [[ "$kdf" == "1" ]]; then + kdf="pbkdf2" + fi _validate_variable_str "kdf" "none pbkdf2" } validate_base_salt() { - if [[ "$kdf" != "none" ]]; then - if [[ "$base_salt" == "" ]]; then - die 1 'The base salt is required when using a kdf, but it is unspecified.' - elif [[ "$base_salt" == "random" ]]; then - die 1 'The base=salt is set to random, but it has not been randomized'. - elif [[ "$base_salt" == "password" ]]; then - die 1 'Using base-salt="password" method with a KDF is insecure. Use "random" instead'. - fi - fi + if [[ "$kdf" != "none" ]]; then + if [[ "$base_salt" == "" ]]; then + die 1 'The base salt is required when using a kdf, but it is unspecified.' + elif [[ "$base_salt" == "random" ]]; then + die 1 'The base=salt is set to random, but it has not been randomized'. + elif [[ "$base_salt" == "password" ]]; then + die 1 'Using base-salt="password" method with a KDF is insecure. Use "random" instead'. + fi + fi } get_use_versioned_config() { - #If the user did not specify to use the versioned config or not, then: - # 1. Check if there is a setting in the unversioned config. - # 2. If not, check if the versioned config exists. - # 3. If not, then defer further checking until we know our setting of base-salt. + #If the user did not specify to use the versioned config or not, then: + # 1. Check if there is a setting in the unversioned config. + # 2. If not, check if the versioned config exists. + # 3. If not, then defer further checking until we know our setting of base-salt. if [[ "$use_versioned_config" == "" ]]; then use_versioned_config=$(_load_unversioned_config "transcrypt.use-versioned-config" || echo "") fi if [[ "$use_versioned_config" == "" ]]; then - if [ -f "${VERSIONED_TC_CONFIG}" ]; then - use_versioned_config="1" - else - use_versioned_config="defer" # defer to after base-salt setting is read - fi - fi + if [ -f "${VERSIONED_TC_CONFIG}" ]; then + use_versioned_config="1" + else + use_versioned_config="defer" # defer to after base-salt setting is read + fi + fi } # ensure we have a digest to hash the salted password @@ -644,62 +644,62 @@ get_kdf() { } get_base_salt() { - #+-----------+-------------+------------------+--------------------+------------------------+ - #| Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | - #|-----------+-------------+------------------+--------------------+------------------------| - #| configure | True | True | True | Use Argv and Overwrite | - #| configure | True | True | False | Use Argv and Overwrite | - #| configure | True | False | True | Use Config Value | - #| configure | True | False | False | Randomize | - #| configure | False | True | True | Ignored | - #| configure | False | True | False | Ignored | - #| configure | False | False | True | Ignored | - #| configure | False | False | False | Ignored | - #+-----------+-------------+------------------+--------------------+------------------------+ - #+----------+-------------+------------------+--------------------+------------------------+ - #| Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | - #|----------+-------------+------------------+--------------------+------------------------| - #| rekey | True | True | True | Use Argv and Overwrite | - #| rekey | True | True | False | Use Argv and Overwrite | - #| rekey | True | False | True | Randomize | - #| rekey | True | False | False | Randomize | - #| rekey | False | True | True | Ignored | - #| rekey | False | True | False | Ignored | - #| rekey | False | False | True | Ignored | - #| rekey | False | False | False | Ignored | - #+----------+-------------+------------------+--------------------+------------------------+ - # When calling `get_base_salt`, the action is always configure or rekey. - if [[ "$kdf" == "none" ]]; then - if [[ "$base_salt" == "" ]]; then - base_salt="password" - fi - if [[ "$use_versioned_config" == "defer" ]]; then - # Handle deferred use_version_config - use_versioned_config="0" - fi - else - if [[ "$base_salt" == "" ]]; then - # The user did not specify a base salt. - if [[ $rekey ]]; then - base_salt="random" - else - base_salt=$(_load_config_var "transcrypt.base-salt" || echo "random") - fi - else - # The user specified something for base-salt - if [[ "$base_salt" == "password" ]]; then - die 1 'Using the "password" method with a KDF is insecure. Use "random" instead'. - fi - fi - # Randomize the base salt if requested. - if [[ "$base_salt" == "random" ]]; then - base_salt=$(openssl rand -hex 32) - fi - if [[ "$use_versioned_config" == "defer" ]]; then - # Handle deferred use_version_config - use_versioned_config="1" - fi - fi + #+-----------+-------------+------------------+--------------------+------------------------+ + #| Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | + #|-----------+-------------+------------------+--------------------+------------------------| + #| configure | True | True | True | Use Argv and Overwrite | + #| configure | True | True | False | Use Argv and Overwrite | + #| configure | True | False | True | Use Config Value | + #| configure | True | False | False | Randomize | + #| configure | False | True | True | Ignored | + #| configure | False | True | False | Ignored | + #| configure | False | False | True | Ignored | + #| configure | False | False | False | Ignored | + #+-----------+-------------+------------------+--------------------+------------------------+ + #+----------+-------------+------------------+--------------------+------------------------+ + #| Action | Using_KDF | BaseSaltInArgv | BaseSaltInConfig | Outcome | + #|----------+-------------+------------------+--------------------+------------------------| + #| rekey | True | True | True | Use Argv and Overwrite | + #| rekey | True | True | False | Use Argv and Overwrite | + #| rekey | True | False | True | Randomize | + #| rekey | True | False | False | Randomize | + #| rekey | False | True | True | Ignored | + #| rekey | False | True | False | Ignored | + #| rekey | False | False | True | Ignored | + #| rekey | False | False | False | Ignored | + #+----------+-------------+------------------+--------------------+------------------------+ + # When calling `get_base_salt`, the action is always configure or rekey. + if [[ "$kdf" == "none" ]]; then + if [[ "$base_salt" == "" ]]; then + base_salt="password" + fi + if [[ "$use_versioned_config" == "defer" ]]; then + # Handle deferred use_version_config + use_versioned_config="0" + fi + else + if [[ "$base_salt" == "" ]]; then + # The user did not specify a base salt. + if [[ $rekey ]]; then + base_salt="random" + else + base_salt=$(_load_config_var "transcrypt.base-salt" || echo "random") + fi + else + # The user specified something for base-salt + if [[ "$base_salt" == "password" ]]; then + die 1 'Using the "password" method with a KDF is insecure. Use "random" instead'. + fi + fi + # Randomize the base salt if requested. + if [[ "$base_salt" == "random" ]]; then + base_salt=$(openssl rand -hex 32) + fi + if [[ "$use_versioned_config" == "defer" ]]; then + # Handle deferred use_version_config + use_versioned_config="1" + fi + fi } # ensure we have a password to encrypt with @@ -904,14 +904,14 @@ display_configuration() { printf 'Copy and paste the following command to initialize a cloned repository:\n\n' if [[ "$use_versioned_config" == "0" ]] && [[ "$kdf" != "none" ]] ; then - # The user is forcing the versioned config off, give them an invocation - # that respects that - printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s' -vc '%s'\n" \ - "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" "$use_versioned_config" - else - printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s'\n" \ - "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" - fi + # The user is forcing the versioned config off, give them an invocation + # that respects that + printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s' -vc '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" "$use_versioned_config" + else + printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s'\n" \ + "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" + fi } # remove transcrypt-related settings from the repository's git config @@ -1266,24 +1266,24 @@ help() { -md, --digest=DIGEST the digest used to hash the salted password; defaults to md5. It is strongly recommended to use - a stronger hash (e.g. sha256) if possible. + a stronger hash (e.g. sha256) if possible. --kdf=PBKDF2 the key-derivation-function to use. Currently can be either - 'pbkdf2' or 'none'. Defaults to none. It is strongly - recommended to use a kdf (e.g. pbkdf2) if possible. + 'pbkdf2' or 'none'. Defaults to none. It is strongly + recommended to use a kdf (e.g. pbkdf2) if possible. -pbkdf2 equivalent to passing --kdf2='pbkdf2' -bs, --base-salt=BASE_SALT - if specified, and a KDF is used, this overrides the text - used as the basis for salt-generation. + if specified, and a KDF is used, this overrides the text + used as the basis for salt-generation. -vc, --versioned-config=<0|1> - Force transcrypt to use or not use the versioned config. - By default the versioned config is only used when using a - KDF. + Force transcrypt to use or not use the versioned config. + By default the versioned config is only used when using a + KDF. --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH @@ -1393,7 +1393,7 @@ transcrypt_main() { kdf='' digest='' base_salt='' - use_versioned_config='' + use_versioned_config='' # used to bypass certain safety checks requires_existing_config='' @@ -1567,8 +1567,8 @@ transcrypt_main() { # always run our safety checks run_safety_checks - # determine if we are using the versioned config or not - get_use_versioned_config + # determine if we are using the versioned config or not + get_use_versioned_config # regular expression used to test user input readonly YES_REGEX='^[Yy]$' From dc5eec5fddcf099c5dfea21b982178fa2e3a307b Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 18:00:09 -0400 Subject: [PATCH 20/38] Update completion scripts --- contrib/bash/transcrypt | 14 +++++++++----- contrib/zsh/_transcrypt | 1 + man/transcrypt.1 | 14 +++++++++++++- man/transcrypt.1.ronn | 17 +++++++++++++++++ transcrypt | 11 +++++------ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/contrib/bash/transcrypt b/contrib/bash/transcrypt index 2e02fbd..4db57c8 100644 --- a/contrib/bash/transcrypt +++ b/contrib/bash/transcrypt @@ -21,8 +21,8 @@ _transcrypt() { COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - opts="-c -p -y -d -r -f -F -u -l -s -e -i -v -h \ - --cipher --password --set-openssl-path --yes --display --rekey --flush-credentials --force --uninstall --upgrade --list --show-raw --export-gpg --import-gpg --version --help" + opts="-c -p -md -pbkdf2 -bs -vc -y -d -r -f -F -u -l -s -e -i -v -h \ + --cipher --password --digest --kdf --base-salt --versioned-config --set-openssl-path --yes --display --rekey --flush-credentials --force --uninstall --upgrade --list --show-raw --export-gpg --import-gpg --version --help" case "${prev}" in -c | --cipher) @@ -34,16 +34,20 @@ _transcrypt() { return 0 ;; -md | --digest) - local ciphers=$(openssl list-digest-commands || openssl list -digest-commands &2>/dev/null) - COMPREPLY=( $(compgen -W "${ciphers}" -- ${cur}) ) + local digests=$(openssl list-digest-commands || openssl list -digest-commands &2>/dev/null) + COMPREPLY=( $(compgen -W "${digests}" -- ${cur}) ) return 0 ;; - --kdf) + --kdf) + COMPREPLY=( $(compgen -W "none pbkdf2" -- ${cur}) ) return 0 ;; -bs | --base-salt) return 0 ;; + -vc | --versioned-config) + return 0 + ;; -s | --show-raw) _files_and_dirs return 0 diff --git a/contrib/zsh/_transcrypt b/contrib/zsh/_transcrypt index e4aa28e..0227265 100644 --- a/contrib/zsh/_transcrypt +++ b/contrib/zsh/_transcrypt @@ -14,6 +14,7 @@ _transcrypt() { '(-md --digest -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify encryption digest]:digest' \ '(-bs --base-salt -d --display -f --flush-credentials -u --uninstall)'{-bs,--base-salt=}'[specify base-salt]:base-salt' \ '(-pbkdf2 --kdf -d --display -f --flush-credentials -u --uninstall)'{-pbkdf2,--kdf=}'[specify kdf]:kdf' \ + '(-vc --versioned-config -d --display -f --flush-credentials -u --uninstall)'{-vc,--versioned-config=}'[specify use-versioned-config]:use-versioned-config' \ '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ '(-d --display -p --password -c --cipher -r --rekey -u --uninstall)'{-d,--display}'[display current credentials]' \ diff --git a/man/transcrypt.1 b/man/transcrypt.1 index 0ab304a..174d43e 100644 --- a/man/transcrypt.1 +++ b/man/transcrypt.1 @@ -1,6 +1,6 @@ .\" generated with Ronn-NG/v0.9.1 .\" http://github.com/apjanke/ronn-ng/tree/0.9.1 -.TH "TRANSCRYPT" "1" "June 2022" "" +.TH "TRANSCRYPT" "1" "July 2022" "" .SH "NAME" \fBtranscrypt\fR \- transparently encrypt files within a git repository .SH "SYNOPSIS" @@ -17,6 +17,18 @@ the symmetric cipher to utilize for encryption; defaults to aes\-256\-cbc \fB\-p\fR, \fB\-\-password\fR=\fIpassword\fR the password to derive the key from; defaults to 30 random base64 characters .TP +\fB\-md\fR, \fB\-\-digest\fR=\fIdigest\fR +the password to derive the key from; defaults to 30 random base64 characters +.TP +\fB\-\-kdf\fR=\fIkdf\fR +the digest used to hash the salted password; defaults to md5\. It is strongly recommended to use a stronger hash (e\.g\. sha256) if possible\. +.TP +\fB\-bs\fR, \fB\-\-base\-salt\fR=\fIbase_salt\fR +if specified, and a KDF is used, this overrides the text used as the basis for salt\-generation\. +.TP +\fB\-vs\fR, \fB\-\-versioned\-config\fR=\fIuse_versioned_config\fR +force transcrypt to use or not use the versioned config\. By default the versioned config is used when using a KDF\. +.TP \fB\-\-set\-openssl\-path\fR=\fIpath_to_openssl\fR use OpenSSL at this path; defaults to 'openssl' in $PATH .TP diff --git a/man/transcrypt.1.ronn b/man/transcrypt.1.ronn index ef0d072..f928193 100644 --- a/man/transcrypt.1.ronn +++ b/man/transcrypt.1.ronn @@ -25,6 +25,23 @@ The transcrypt source code and full documentation may be downloaded from the password to derive the key from; defaults to 30 random base64 characters + * `-md`, `--digest`=: + the password to derive the key from; + defaults to 30 random base64 characters + + * `--kdf`=: + the digest used to hash the salted password; + defaults to md5. It is strongly recommended to use + a stronger hash (e.g. sha256) if possible. + + * `-bs`, `--base-salt`=: + if specified, and a KDF is used, this overrides the text + used as the basis for salt-generation. + + * `-vc`, `--versioned-config`=: + force transcrypt to use or not use the versioned config. + By default the versioned config is used when using a KDF. + * `--set-openssl-path`=: use OpenSSL at this path; defaults to 'openssl' in $PATH diff --git a/transcrypt b/transcrypt index 7267514..fbe77f3 100755 --- a/transcrypt +++ b/transcrypt @@ -1269,9 +1269,9 @@ help() { a stronger hash (e.g. sha256) if possible. --kdf=PBKDF2 - the key-derivation-function to use. Currently can be either - 'pbkdf2' or 'none'. Defaults to none. It is strongly - recommended to use a kdf (e.g. pbkdf2) if possible. + the key-derivation-function to use. Can be either 'pbkdf2' + or 'none'. Defaults to none. It is strongly recommended to + use a kdf (e.g. pbkdf2) if possible. -pbkdf2 equivalent to passing --kdf2='pbkdf2' @@ -1281,9 +1281,8 @@ help() { used as the basis for salt-generation. -vc, --versioned-config=<0|1> - Force transcrypt to use or not use the versioned config. - By default the versioned config is only used when using a - KDF. + force transcrypt to use or not use the versioned config. + By default the versioned config is only when using a KDF. --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH From d5006d71bc570cd487088285e96a72e559f9edf3 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 18:00:44 -0400 Subject: [PATCH 21/38] reset editable install to 0 --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index fbe77f3..b0a7bd5 100755 --- a/transcrypt +++ b/transcrypt @@ -36,7 +36,7 @@ readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path # Set to 1 to enable a development editable installation # This will symlink transcrypt into the .git folder instead of copying it -readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=1} +readonly EDITABLE_INSTALL=${TRANSCRYPT_EDITABLE_INSTALL:=0} ##### FUNCTIONS From de86fa7a49d6a83ae1fdf263f7dfca43d899fecb Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 3 Jul 2022 18:01:40 -0400 Subject: [PATCH 22/38] Add use-versioned-config to versioned blocklist --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index b0a7bd5..5f47a83 100755 --- a/transcrypt +++ b/transcrypt @@ -32,7 +32,7 @@ readonly DEFAULT_KDF='none' # These are config variables we do not allow to be used in the versioned # configuration -readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path" +readonly VERSIONED_CONFIG_BLOCKLIST="transcrypt.password transcrypt.openssl-path transcrypt.use-versioned-config" # Set to 1 to enable a development editable installation # This will symlink transcrypt into the .git folder instead of copying it From 39dba95900d8f5e01fbb081b13287e85aa6518c5 Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 8 Jul 2022 13:47:09 -0400 Subject: [PATCH 23/38] Fix bad function call --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index 5f47a83..768d074 100755 --- a/transcrypt +++ b/transcrypt @@ -603,7 +603,7 @@ get_use_versioned_config() { # 2. If not, check if the versioned config exists. # 3. If not, then defer further checking until we know our setting of base-salt. if [[ "$use_versioned_config" == "" ]]; then - use_versioned_config=$(_load_unversioned_config "transcrypt.use-versioned-config" || echo "") + use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config" || echo "") fi if [[ "$use_versioned_config" == "" ]]; then if [ -f "${VERSIONED_TC_CONFIG}" ]; then From d10800ac8937288ae04ff1b8ca6f2ba1760052ce Mon Sep 17 00:00:00 2001 From: James Murty Date: Sat, 9 Jul 2022 22:17:19 +1000 Subject: [PATCH 24/38] Comply with shfmt --- transcrypt | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/transcrypt b/transcrypt index 768d074..2e14431 100755 --- a/transcrypt +++ b/transcrypt @@ -596,11 +596,10 @@ validate_base_salt() { fi } - get_use_versioned_config() { #If the user did not specify to use the versioned config or not, then: - # 1. Check if there is a setting in the unversioned config. - # 2. If not, check if the versioned config exists. + # 1. Check if there is a setting in the unversioned config. + # 2. If not, check if the versioned config exists. # 3. If not, then defer further checking until we know our setting of base-salt. if [[ "$use_versioned_config" == "" ]]; then use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config" || echo "") @@ -609,7 +608,7 @@ get_use_versioned_config() { if [ -f "${VERSIONED_TC_CONFIG}" ]; then use_versioned_config="1" else - use_versioned_config="defer" # defer to after base-salt setting is read + use_versioned_config="defer" # defer to after base-salt setting is read fi fi } @@ -903,7 +902,7 @@ display_configuration() { _display_runtime_configuration printf 'Copy and paste the following command to initialize a cloned repository:\n\n' - if [[ "$use_versioned_config" == "0" ]] && [[ "$kdf" != "none" ]] ; then + if [[ "$use_versioned_config" == "0" ]] && [[ "$kdf" != "none" ]]; then # The user is forcing the versioned config off, give them an invocation # that respects that printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s' -vc '%s'\n" \ @@ -1270,8 +1269,8 @@ help() { --kdf=PBKDF2 the key-derivation-function to use. Can be either 'pbkdf2' - or 'none'. Defaults to none. It is strongly recommended to - use a kdf (e.g. pbkdf2) if possible. + or 'none'. Defaults to none. It is strongly recommended to + use a kdf (e.g. pbkdf2) if possible. -pbkdf2 equivalent to passing --kdf2='pbkdf2' @@ -1282,7 +1281,7 @@ help() { -vc, --versioned-config=<0|1> force transcrypt to use or not use the versioned config. - By default the versioned config is only when using a KDF. + By default the versioned config is only when using a KDF. --set-openssl-path=PATH_TO_OPENSSL use OpenSSL at this path; defaults to 'openssl' in \$PATH From 794934cb3d46b40d2cb4f629d723032ff4fa5bc0 Mon Sep 17 00:00:00 2001 From: James Murty Date: Sat, 9 Jul 2022 22:21:53 +1000 Subject: [PATCH 25/38] Fix tests after change to base salt -bs argument --- tests/test_init.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_init.bats b/tests/test_init.bats index 07eade3..47b7c37 100755 --- a/tests/test_init.bats +++ b/tests/test_init.bats @@ -61,7 +61,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] } @test "init: show details for -d" { @@ -73,7 +73,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] } @test "init: respects core.hooksPath setting" { @@ -90,7 +90,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -sm 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] } @test "init: transcrypt.openssl-path config setting defaults to 'openssl'" { From f6c100e06d0087180ab67ddb92506678a3b66a09 Mon Sep 17 00:00:00 2001 From: James Murty Date: Sat, 9 Jul 2022 22:31:33 +1000 Subject: [PATCH 26/38] Display of runtime config shows digest, and kdf & base salt only when relevant --- tests/test_init.bats | 3 +++ transcrypt | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/test_init.bats b/tests/test_init.bats index 47b7c37..6e85b7b 100755 --- a/tests/test_init.bats +++ b/tests/test_init.bats @@ -60,6 +60,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [ "$status" -eq 0 ] [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] + [[ "${output}" = *" DIGEST: md5"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] } @@ -72,6 +73,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [ "$status" -eq 0 ] [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] + [[ "${output}" = *" DIGEST: md5"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] } @@ -89,6 +91,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [ "$status" -eq 0 ] [[ "${output}" = *"The current repository was configured using transcrypt version $VERSION"* ]] [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] + [[ "${output}" = *" DIGEST: md5"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] } diff --git a/transcrypt b/transcrypt index 2e14431..31e45ec 100755 --- a/transcrypt +++ b/transcrypt @@ -885,11 +885,19 @@ _display_git_configuration() { # Show the config of the current runtime _display_runtime_configuration() { - printf ' DIGEST: %s\n' "$digest" - printf ' KDF: %s\n' "$kdf" - printf ' BASE_SALT: %s\n' "$base_salt" - printf ' CIPHER: %s\n' "$cipher" - printf ' PASSWORD: %s\n\n' "$password" + if [[ "$kdf" == "none" ]]; then + # KDF is not configured, no need to show KDF or base salt config + printf ' CIPHER: %s\n' "$cipher" + printf ' DIGEST: %s\n' "$digest" + printf ' PASSWORD: %s\n' "$password" + else + # KDF is configured, show KDF and base salt config + printf ' CIPHER: %s\n' "$cipher" + printf ' DIGEST: %s\n' "$digest" + printf ' KDF: %s\n' "$kdf" + printf ' PASSWORD: %s\n' "$password" + printf ' BASE_SALT: %s\n\n' "$base_salt" + fi } # display the current configuration settings From b16de589ad11e0cbf2bf5eea932c21e84462c888 Mon Sep 17 00:00:00 2001 From: James Murty Date: Sat, 9 Jul 2022 23:54:43 +1000 Subject: [PATCH 27/38] Accept `-k` argument as short-hand for `--kdf` --- contrib/bash/transcrypt | 2 +- contrib/zsh/_transcrypt | 2 +- transcrypt | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/bash/transcrypt b/contrib/bash/transcrypt index 4db57c8..2474566 100644 --- a/contrib/bash/transcrypt +++ b/contrib/bash/transcrypt @@ -21,7 +21,7 @@ _transcrypt() { COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - opts="-c -p -md -pbkdf2 -bs -vc -y -d -r -f -F -u -l -s -e -i -v -h \ + opts="-c -p -md -k -bs -vc -y -d -r -f -F -u -l -s -e -i -v -h \ --cipher --password --digest --kdf --base-salt --versioned-config --set-openssl-path --yes --display --rekey --flush-credentials --force --uninstall --upgrade --list --show-raw --export-gpg --import-gpg --version --help" case "${prev}" in diff --git a/contrib/zsh/_transcrypt b/contrib/zsh/_transcrypt index 0227265..56152dd 100644 --- a/contrib/zsh/_transcrypt +++ b/contrib/zsh/_transcrypt @@ -13,7 +13,7 @@ _transcrypt() { '(-c --cipher -d --display -f --flush-credentials -u --uninstall)'{-c,--cipher=}'[specify encryption cipher]:cipher:->cipher' \ '(-md --digest -d --display -f --flush-credentials -u --uninstall)'{-md,--digest=}'[specify encryption digest]:digest' \ '(-bs --base-salt -d --display -f --flush-credentials -u --uninstall)'{-bs,--base-salt=}'[specify base-salt]:base-salt' \ - '(-pbkdf2 --kdf -d --display -f --flush-credentials -u --uninstall)'{-pbkdf2,--kdf=}'[specify kdf]:kdf' \ + '(-k --kdf -d --display -f --flush-credentials -u --uninstall)'{-k,--kdf=}'[specify kdf]:kdf' \ '(-vc --versioned-config -d --display -f --flush-credentials -u --uninstall)'{-vc,--versioned-config=}'[specify use-versioned-config]:use-versioned-config' \ '(-p --password -d --display -f --flush-credentials -u --uninstall)'{-p,--password=}'[specify encryption password]:password:' \ '(-y --yes)'{-y,--yes}'[assume yes and accept defaults]' \ diff --git a/transcrypt b/transcrypt index 31e45ec..e338728 100755 --- a/transcrypt +++ b/transcrypt @@ -913,10 +913,10 @@ display_configuration() { if [[ "$use_versioned_config" == "0" ]] && [[ "$kdf" != "none" ]]; then # The user is forcing the versioned config off, give them an invocation # that respects that - printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s' -vc '%s'\n" \ + printf " transcrypt -c '%s' -p '%s' -md '%s' -k '%s' -bs '%s' -vc '%s'\n" \ "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" "$use_versioned_config" else - printf " transcrypt -c '%s' -p '%s' -md '%s' --kdf '%s' -bs '%s'\n" \ + printf " transcrypt -c '%s' -p '%s' -md '%s' -k '%s' -bs '%s'\n" \ "$cipher" "$escaped_password" "$digest" "$kdf" "$base_salt" fi } @@ -1275,7 +1275,7 @@ help() { defaults to md5. It is strongly recommended to use a stronger hash (e.g. sha256) if possible. - --kdf=PBKDF2 + -k, --kdf=PBKDF2 the key-derivation-function to use. Can be either 'pbkdf2' or 'none'. Defaults to none. It is strongly recommended to use a kdf (e.g. pbkdf2) if possible. @@ -1451,7 +1451,7 @@ transcrypt_main() { -pbkdf2) kdf=pbkdf2 ;; - --kdf) + -k | --kdf) kdf=${2} shift ;; From f39759b8e5ffe96dbc6a6e0487acdba71bfb4852 Mon Sep 17 00:00:00 2001 From: James Murty Date: Sat, 9 Jul 2022 23:59:39 +1000 Subject: [PATCH 28/38] Remove `-pbkdf2` shorthand argument for `-k pbkdf2` We don't have algorithm-specific shorthands for other things like cipher, and if transcrypt will support multiple potential KDFs it will be unwieldy to add arguments for every one. --- tests/local_test.sh | 2 +- transcrypt | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/local_test.sh b/tests/local_test.sh index 51f1d42..4c70b51 100644 --- a/tests/local_test.sh +++ b/tests/local_test.sh @@ -1,5 +1,5 @@ #./transcrypt -F -c aes-256-cbc -p "foobar" -md SHA512 -sm configured --use_pbkdf2=0 -./transcrypt -F -c aes-256-cbc -pbkdf2 -p "foobar" -md SHA512 -sm configured +./transcrypt -F -c aes-256-cbc -k pbkdf2 -p "foobar" -md SHA512 -sm configured ./transcrypt -d diff --git a/transcrypt b/transcrypt index e338728..07d34d9 100755 --- a/transcrypt +++ b/transcrypt @@ -1280,9 +1280,6 @@ help() { or 'none'. Defaults to none. It is strongly recommended to use a kdf (e.g. pbkdf2) if possible. - -pbkdf2 - equivalent to passing --kdf2='pbkdf2' - -bs, --base-salt=BASE_SALT if specified, and a KDF is used, this overrides the text used as the basis for salt-generation. @@ -1448,9 +1445,6 @@ transcrypt_main() { --digest=*) digest=${1#*=} ;; - -pbkdf2) - kdf=pbkdf2 - ;; -k | --kdf) kdf=${2} shift From dc735060b169e0db5bc5e0b1123c515aaf2419e8 Mon Sep 17 00:00:00 2001 From: James Murty Date: Sun, 10 Jul 2022 00:26:48 +1000 Subject: [PATCH 29/38] Fix tests to use `-k` instead of `--kdf` --- tests/test_init.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_init.bats b/tests/test_init.bats index 6e85b7b..0e94b2f 100755 --- a/tests/test_init.bats +++ b/tests/test_init.bats @@ -62,7 +62,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" DIGEST: md5"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' -k 'none' -bs 'password'"* ]] } @test "init: show details for -d" { @@ -75,7 +75,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" DIGEST: md5"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' -k 'none' -bs 'password'"* ]] } @test "init: respects core.hooksPath setting" { @@ -93,7 +93,7 @@ SETUP_SKIP_INIT_TRANSCRYPT=1 [[ "${output}" = *" CIPHER: aes-256-cbc"* ]] [[ "${output}" = *" DIGEST: md5"* ]] [[ "${output}" = *" PASSWORD: abc 123"* ]] - [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' --kdf 'none' -bs 'password'"* ]] + [[ "${output}" = *" transcrypt -c 'aes-256-cbc' -p 'abc 123' -md 'md5' -k 'none' -bs 'password'"* ]] } @test "init: transcrypt.openssl-path config setting defaults to 'openssl'" { From ef1cb90fcd36250c59c6717e9befe89dfa3a852d Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 10 Jul 2022 14:37:12 -0400 Subject: [PATCH 30/38] Add logic to determine what kdfs are available --- tests/test_transcrypt.py | 18 +++++++++++++++++- transcrypt | 29 ++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index ac2b52c..e4281bd 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -613,6 +613,22 @@ def test_salt_changes_when_kdf_changes(): assert config4['base_salt'] == 'password' +def test_unsupported_kdf(): + config = { + 'cipher': 'aes-256-cbc', + 'password': 'correct horse battery staple', + 'digest': 'sha512', + 'kdf': 'MY_FAVORITE_UNSUPPORTED_KDF', + 'base_salt': None, + } + verbose = 2 + self = TestCases(config=config, verbose=verbose) + import subprocess + import pytest + with pytest.raises(subprocess.CalledProcessError): + self.setup() + + def test_kdf_setting_preserved_on_rekey(): config = { 'cipher': 'aes-256-cbc', @@ -648,7 +664,7 @@ def test_configuration_grid(): 'cipher': ['aes-256-cbc', 'aes-128-ecb'], 'password': ['correct horse battery staple'], 'digest': ['md5', 'sha256'], - 'kdf': ['none', 'pbkdf2'], + 'kdf': ['none', 'pbkdf2', 'scrypt'], 'base_salt': ['password', 'random', 'mylittlecustomsalt', None], 'use_versioned_config': ['0', '1', None], } diff --git a/transcrypt b/transcrypt index 07d34d9..4c96500 100755 --- a/transcrypt +++ b/transcrypt @@ -257,7 +257,7 @@ _load_vars_for_encryption() { # Helper to populate variables needed to call openssl encryption _load_transcrypt_config_vars - if [[ "$kdf" == "1" ]] || [[ "$kdf" == "pbkdf2" ]]; then + if [[ "$kdf" == "pbkdf2" ]]; then pbkdf2_arg='-pbkdf2' fi @@ -576,12 +576,27 @@ validate_digest() { validate_kdf() { # Normalize kdf kdf=$(echo "${kdf}" | tr '[:upper:]' '[:lower:]') - if [[ "$kdf" == "0" ]]; then - kdf="none" - elif [[ "$kdf" == "1" ]]; then - kdf="pbkdf2" - fi - _validate_variable_str "kdf" "none pbkdf2" + + # References: + # https://wiki.openssl.org/index.php/EVP_Key_Derivation + openssl_version=$($openssl_path version | cut -d' ' -f2) + openssl_major_version=$(echo "$openssl_version" | cut -d'.' -f1) + openssl_minor_version=$(echo "$openssl_version" | cut -d'.' -f2) + if [ "$openssl_major_version" -ge "3" ]; then + # If openssl is >= 3 we can list kdf algos, via _openssl_list kdf-algorithms + # however the output format is a pain to parse in bash, so we hard code + # known KDFs that we support + valid_kdfs="none pbkdf2 scrypt" + else + if [ "$openssl_minor_version" -ge "1" ]; then + # In 1.1.x pbkdf2 is the only kdf available + valid_kdfs="none pbkdf2" + else + # In <1.0.x there are no kdfs available + valid_kdfs="none" + fi + fi + _validate_variable_str "kdf" "$valid_kdfs" } validate_base_salt() { From a459c82c4d64d1065d3f0eaf11b569ce613d7657 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 10 Jul 2022 14:45:27 -0400 Subject: [PATCH 31/38] disable scrypt --- tests/test_transcrypt.py | 2 +- transcrypt | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_transcrypt.py b/tests/test_transcrypt.py index e4281bd..29705e7 100644 --- a/tests/test_transcrypt.py +++ b/tests/test_transcrypt.py @@ -664,7 +664,7 @@ def test_configuration_grid(): 'cipher': ['aes-256-cbc', 'aes-128-ecb'], 'password': ['correct horse battery staple'], 'digest': ['md5', 'sha256'], - 'kdf': ['none', 'pbkdf2', 'scrypt'], + 'kdf': ['none', 'pbkdf2'], 'base_salt': ['password', 'random', 'mylittlecustomsalt', None], 'use_versioned_config': ['0', '1', None], } diff --git a/transcrypt b/transcrypt index 4c96500..8b07e94 100755 --- a/transcrypt +++ b/transcrypt @@ -586,7 +586,11 @@ validate_kdf() { # If openssl is >= 3 we can list kdf algos, via _openssl_list kdf-algorithms # however the output format is a pain to parse in bash, so we hard code # known KDFs that we support - valid_kdfs="none pbkdf2 scrypt" + # + # Note: it is not straightforward on how to pass these to openssl enc + # except for the case of pbkdf2, disable others until we figure it out + #valid_kdfs="none pbkdf2 scrypt" + valid_kdfs="none pbkdf2" else if [ "$openssl_minor_version" -ge "1" ]; then # In 1.1.x pbkdf2 is the only kdf available From eeb3160a6694cb2f51e1948bcf206827f9597c4d Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 10 Jul 2022 14:56:35 -0400 Subject: [PATCH 32/38] Change default random salt length to 16 --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index 8b07e94..529f015 100755 --- a/transcrypt +++ b/transcrypt @@ -711,7 +711,7 @@ get_base_salt() { fi # Randomize the base salt if requested. if [[ "$base_salt" == "random" ]]; then - base_salt=$(openssl rand -hex 32) + base_salt=$(openssl rand -hex 16) fi if [[ "$use_versioned_config" == "defer" ]]; then # Handle deferred use_version_config From ce60809fce867efbc18ed444fda0f61d781ed0bb Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 11 Jul 2022 11:23:45 -0400 Subject: [PATCH 33/38] Use fix from #119 --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index 529f015..fe31613 100755 --- a/transcrypt +++ b/transcrypt @@ -378,7 +378,7 @@ git_clean() { else _load_vars_for_encryption # NOTE: the openssl standard for salt is 16 hex bytes. - final_salt=$("$openssl_path" dgst -hmac "${filename}:${extra_salt}" -sha256 "$filename" | tr -d '\r\n' | tail -c 16) + final_salt=$("$openssl_path" dgst -hmac "${filename}:${extra_salt}" -sha256 "$tempfile" | tr -d '\r\n' | tail -c 16) _openssl_encrypt fi } From b3aff0a352fdadf625fc69f187aa648eba8f2add Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 11 Jul 2022 12:41:01 -0400 Subject: [PATCH 34/38] Fallback to not using the versioned config --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index fe31613..c57f410 100755 --- a/transcrypt +++ b/transcrypt @@ -232,7 +232,7 @@ _set_config_var() { # shellcheck disable=SC2155 _load_transcrypt_config_vars() { # Populate bash vars with our config - use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config") || (echo "failed to load transcrypt.use-versioned-config" && false) + use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config") || (echo "0") cipher=$(_load_config_var "transcrypt.cipher") || (echo "failed to load transcrypt.cipher" && false) digest=$(_load_config_var "transcrypt.digest") || (echo "failed to load transcrypt.digest" && false) kdf=$(_load_config_var "transcrypt.kdf") || (echo "failed to load transcrypt.kdf" && false) From 6d7b49812d0d3ff360f50e7b63c114f654e7b213 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 11 Jul 2022 12:45:03 -0400 Subject: [PATCH 35/38] Changed my mind, fallback to using the versioned config (which should be the same as using it if it exists) --- transcrypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transcrypt b/transcrypt index c57f410..d63bccd 100755 --- a/transcrypt +++ b/transcrypt @@ -232,7 +232,7 @@ _set_config_var() { # shellcheck disable=SC2155 _load_transcrypt_config_vars() { # Populate bash vars with our config - use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config") || (echo "0") + use_versioned_config=$(_load_unversioned_config_var "transcrypt.use-versioned-config") || use_versioned_config=1 cipher=$(_load_config_var "transcrypt.cipher") || (echo "failed to load transcrypt.cipher" && false) digest=$(_load_config_var "transcrypt.digest") || (echo "failed to load transcrypt.digest" && false) kdf=$(_load_config_var "transcrypt.kdf") || (echo "failed to load transcrypt.kdf" && false) From f58467e1abeb80607e6b9b1c41a3967430939cfb Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 21 Aug 2022 00:53:15 -0400 Subject: [PATCH 36/38] start work on a multi-image-docker test --- example/end_to_end_example.sh | 1 + tests/oci_container.py | 356 ++++++++++++++++++++++++++++++++++ tests/test_multi_docker.py | 59 ++++++ 3 files changed, 416 insertions(+) create mode 100644 tests/oci_container.py create mode 100644 tests/test_multi_docker.py diff --git a/example/end_to_end_example.sh b/example/end_to_end_example.sh index 2d54834..717f87a 100644 --- a/example/end_to_end_example.sh +++ b/example/end_to_end_example.sh @@ -2,6 +2,7 @@ __doc__=" A simple demo of transcrypt " +set -x TMP_DIR=$HOME/tmp/transcrypt-demo mkdir -p "$TMP_DIR" diff --git a/tests/oci_container.py b/tests/oci_container.py new file mode 100644 index 0000000..39f057f --- /dev/null +++ b/tests/oci_container.py @@ -0,0 +1,356 @@ +""" +Ported from cibuildwheel: https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/oci_container.py +""" +from __future__ import annotations + +import io +import json +import os +import itertools +# import platform +import shlex +import shutil +import subprocess +import sys +import uuid +from pathlib import Path, PurePath, PurePosixPath +from types import TracebackType +from typing import IO, Dict, Sequence, cast, Any +Literal, PathOrStr, PopenBytes = Any, Any, Any +ContainerEngine = Any + + +class OCIContainer: + """ + An object that represents a running OCI (e.g. Docker) container. + + Intended for use as a context manager e.g. + `with OCIContainer(image = 'ubuntu') as docker:` + + A bash shell is running in the remote container. When `call()` is invoked, + the command is relayed to the remote shell, and the results are streamed + back to cibuildwheel. + + Args: + volumes (None | List[Tuple[PathLike, PathLike]]): + A list of (external-path, internal-path) tuples + + Example: + >>> image = 'ubuntu:22.04' + >>> # Test the default container + >>> with OCIContainer(image=image) as self: + ... self.call(["echo", "hello world"]) + ... self.call(["cat", "/proc/1/cgroup"]) + ... print(self.debug_info()) + """ + + UTILITY_PYTHON = "python3" + + process: PopenBytes + bash_stdin: IO[bytes] + bash_stdout: IO[bytes] + + def __init__( + self, + *, + image: str, + simulate_32_bit: bool = False, + cwd: PathOrStr | None = None, + volumes=None, + engine: ContainerEngine = "docker", + ): + if not image: + raise ValueError("Must have a non-empty image to run.") + + self.image = image + self.simulate_32_bit = simulate_32_bit + self.cwd = cwd + self.volumes = volumes + self.name: str | None = None + self.engine = engine + + def start(self): + self.name = f"OCIContainer-{uuid.uuid4()}" + network_args = [] + shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"] + + volume_argvals = ['{}:{}'.format(v1, v2) for v1, v2 in self.volumes] + volume_argkeys = ['--volume'] * len(volume_argvals) + volume_args = list(itertools.chain(*zip(volume_argkeys, volume_argvals))) + print(f'volume_args={volume_args}') + + subprocess.run( + [ + self.engine, + "create", + f"--name={self.name}", + "--interactive", + *network_args, + *volume_args, + self.image, + *shell_args, + ], + check=True, + ) + + self.process = subprocess.Popen( + [ + self.engine, + "start", + "--attach", + "--interactive", + self.name, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + assert self.process.stdin and self.process.stdout + self.bash_stdin = self.process.stdin + self.bash_stdout = self.process.stdout + + # run a noop command to block until the container is responding + self.call(["/bin/true"], cwd="/") + + if self.cwd: + # Although `docker create -w` does create the working dir if it + # does not exist, podman does not. There does not seem to be a way + # to setup a workdir for a container running in podman. + self.call(["mkdir", "-p", os.fspath(self.cwd)], cwd="/") + + def stop(self): + self.bash_stdin.write(b"exit 0\n") + self.bash_stdin.flush() + self.process.wait(timeout=30) + self.bash_stdin.close() + self.bash_stdout.close() + + if self.engine == "podman": + # This works around what seems to be a race condition in the podman + # backend. The full reason is not understood. See PR #966 for a + # discussion on possible causes and attempts to remove this line. + # For now, this seems to work "well enough". + self.process.wait() + + assert isinstance(self.name, str) + + subprocess.run( + [self.engine, "rm", "--force", "-v", self.name], + stdout=subprocess.DEVNULL, + check=False, + ) + self.name = None + + def __enter__(self) -> OCIContainer: + self.start() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.stop() + + def copy_into(self, from_path: Path, to_path: PurePath) -> None: + # `docker cp` causes 'no space left on device' error when + # a container is running and the host filesystem is + # mounted. https://github.com/moby/moby/issues/38995 + # Use `docker exec` instead. + + if from_path.is_dir(): + self.call(["mkdir", "-p", to_path]) + subprocess.run( + f"tar cf - . | {self.engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", + shell=True, + check=True, + cwd=from_path, + ) + else: + with subprocess.Popen( + [ + self.engine, + "exec", + "-i", + str(self.name), + "sh", + "-c", + f"cat > {shell_quote(to_path)}", + ], + stdin=subprocess.PIPE, + ) as exec_process: + exec_process.stdin = cast(IO[bytes], exec_process.stdin) + + with open(from_path, "rb") as from_file: + shutil.copyfileobj(from_file, exec_process.stdin) + + exec_process.stdin.close() + exec_process.wait() + + if exec_process.returncode: + raise subprocess.CalledProcessError( + exec_process.returncode, exec_process.args, None, None + ) + + def copy_out(self, from_path: PurePath, to_path: Path) -> None: + # note: we assume from_path is a dir + to_path.mkdir(parents=True, exist_ok=True) + + if self.engine == "podman": + subprocess.run( + [ + self.engine, + "cp", + f"{self.name}:{from_path}/.", + str(to_path), + ], + check=True, + cwd=to_path, + ) + elif self.engine == "docker": + # There is a bug in docker that prevents a simple 'cp' invocation + # from working https://github.com/moby/moby/issues/38995 + command = f"{self.engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -" + subprocess.run( + command, + shell=True, + check=True, + cwd=to_path, + ) + else: + raise KeyError(self.engine) + + def glob(self, path: PurePosixPath, pattern: str) -> list[PurePosixPath]: + glob_pattern = path.joinpath(pattern) + + path_strings = json.loads( + self.call( + [ + self.UTILITY_PYTHON, + "-c", + f"import sys, json, glob; json.dump(glob.glob({str(glob_pattern)!r}), sys.stdout)", + ], + capture_output=True, + ) + ) + + return [PurePosixPath(p) for p in path_strings] + + def call( + self, + args: Sequence[PathOrStr], + env: dict[str, str] | None = None, + capture_output: bool = False, + cwd: PathOrStr | None = None, + ) -> str: + + if cwd is None: + # Podman does not start the a container in a specific working dir + # so we always need to specify it when making calls. + cwd = self.cwd + + chdir = f"cd {cwd}" if cwd else "" + env_assignments = ( + " ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items()) + if env is not None + else "" + ) + command = " ".join(shlex.quote(str(a)) for a in args) + end_of_message = str(uuid.uuid4()) + + # log the command we're executing + print(f" + {command}") + + # Write a command to the remote shell. First we change the + # cwd, if that's required. Then, we use the `env` utility to run + # `command` inside the specified environment. We use `env` because it + # can cope with spaces and strange characters in the name or value. + # Finally, the remote shell is told to write a footer - this will show + # up in the output so we know when to stop reading, and will include + # the return code of `command`. + self.bash_stdin.write( + bytes( + f"""( + {chdir} + env {env_assignments} {command} + printf "%04d%s\n" $? {end_of_message} + ) + """, + encoding="utf8", + errors="surrogateescape", + ) + ) + self.bash_stdin.flush() + + if capture_output: + output_io: IO[bytes] = io.BytesIO() + else: + output_io = sys.stdout.buffer + + while True: + line = self.bash_stdout.readline() + + if line.endswith(bytes(end_of_message, encoding="utf8") + b"\n"): + # fmt: off + footer_offset = ( + len(line) + - 1 # newline character + - len(end_of_message) # delimiter + - 4 # 4 return code decimals + ) + # fmt: on + return_code_str = line[footer_offset : footer_offset + 4] + return_code = int(return_code_str) + # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + break + else: + output_io.write(line) + + if isinstance(output_io, io.BytesIO): + output = str(output_io.getvalue(), encoding="utf8", errors="surrogateescape") + else: + output = "" + + if return_code != 0: + raise subprocess.CalledProcessError(return_code, args, output) + + return output + + def get_environment(self) -> dict[str, str]: + env = json.loads( + self.call( + [ + self.UTILITY_PYTHON, + "-c", + "import sys, json, os; json.dump(os.environ.copy(), sys.stdout)", + ], + capture_output=True, + ) + ) + return cast(Dict[str, str], env) + + def environment_executor(self, command: list[str], environment: dict[str, str]) -> str: + # used as an EnvironmentExecutor to evaluate commands and capture output + return self.call(command, env=environment, capture_output=True) + + def debug_info(self) -> str: + if self.engine == "podman": + command = f"{self.engine} info --debug" + else: + command = f"{self.engine} info" + completed = subprocess.run( + command, + shell=True, + check=True, + cwd=self.cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + output = str(completed.stdout, encoding="utf8", errors="surrogateescape") + return output + + +def shell_quote(path: PurePath) -> str: + return shlex.quote(os.fspath(path)) diff --git a/tests/test_multi_docker.py b/tests/test_multi_docker.py new file mode 100644 index 0000000..f2c59ed --- /dev/null +++ b/tests/test_multi_docker.py @@ -0,0 +1,59 @@ +def main(): + """ + We are going to setup two docker images, execute transcrypt in each, and + then make sure working on multiple machines with different openssl verions + is handled gracefully + """ + import ubelt as ub + import os + import sys + # Hard coded + transcrypt_repo = ub.Path('$HOME/code/transcrypt').expand() + # Hack sys.path + sys.path.append(os.fspath(transcrypt_repo / 'tests')) + + test_dpath = ub.Path.appdir('transcrypt/tests/') + dpath1 = (test_dpath / 'container1').ensuredir() + dpath2 = (test_dpath / 'container2').ensuredir() + + transcrypt_repo + + from oci_container import OCIContainer + from shlex import split as shsplit + + container1 = OCIContainer( + image='ubuntu:20.04', + volumes=[ + (transcrypt_repo, '/transcrypt'), + (dpath1, '/custom'), + ]) + + container2 = OCIContainer( + image='ubuntu:22.04', + volumes=[ + (transcrypt_repo, '/transcrypt'), + (dpath2, '/custom'), + ]) + + container1.start() + container2.start() + + container1.call(['/bin/ls'], cwd='/custom') + container2.call(['/bin/ls'], cwd='/custom') + + def setup_container(container): + container.call(shsplit('apt-get update -y')) + container.call(shsplit('apt-get install git -y')) + container.call(shsplit('git clone /transcrypt/.git'), cwd='/custom') + container.call(shsplit(f'git config --global user.email "{container.name}@test.com"')) + container.call(shsplit(f'git config --global user.name "{container.name}"')) + container.call(shsplit('git config --global init.defaultBranch "main"')) + container.call(shsplit('ln -s /custom/transcrypt /usr/local/bin/transcrypt'), cwd='/custom') + container.call(shsplit('git status'), cwd='/custom/transcrypt') + container.call(shsplit('bash end_to_end_example.sh'), cwd='/custom/transcrypt/example') + + setup_container(container1) + setup_container(container2) + + container1.call(shsplit('git clone /transcrypt/.git'), cwd='/custom') + container2.call(shsplit('git clone /transcrypt/.git'), cwd='/custom') From 5b464c84cb556024ab09b1b983de0c76eac71be0 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 21 Aug 2022 00:59:13 -0400 Subject: [PATCH 37/38] Fix end-to-end example --- example/end_to_end_example.sh | 2 +- tests/test_multi_docker.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/example/end_to_end_example.sh b/example/end_to_end_example.sh index 717f87a..fa5f339 100644 --- a/example/end_to_end_example.sh +++ b/example/end_to_end_example.sh @@ -30,7 +30,7 @@ mkdir -p "$DEMO_REPO"/safe # Configure transcrypt with a KDF, but an old hash function -transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=1 -y +transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=pbkdf2 -y git commit -m "Configured transcrypt" echo "Secret contents" > "$DEMO_REPO"/safe/secret_file diff --git a/tests/test_multi_docker.py b/tests/test_multi_docker.py index f2c59ed..fe3f7b8 100644 --- a/tests/test_multi_docker.py +++ b/tests/test_multi_docker.py @@ -48,8 +48,14 @@ def setup_container(container): container.call(shsplit(f'git config --global user.email "{container.name}@test.com"')) container.call(shsplit(f'git config --global user.name "{container.name}"')) container.call(shsplit('git config --global init.defaultBranch "main"')) - container.call(shsplit('ln -s /custom/transcrypt /usr/local/bin/transcrypt'), cwd='/custom') + + container.call(shsplit('chmod +x /custom/transcrypt/transcrypt'), cwd='/custom') + container.call(shsplit('ls -al /custom/transcrypt'), cwd='/custom') + container.call(shsplit('ln -s /custom/transcrypt/transcrypt /usr/local/bin/transcrypt'), cwd='/custom') + container.call(shsplit('transcrypt --version'), cwd='/custom/transcrypt/example') + container.call(shsplit('git status'), cwd='/custom/transcrypt') + container.call(shsplit('git pull'), cwd='/custom/transcrypt') container.call(shsplit('bash end_to_end_example.sh'), cwd='/custom/transcrypt/example') setup_container(container1) From 76f00e1ef1f4db026f0b018dbb07af58b601e435 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 21 Aug 2022 01:24:46 -0400 Subject: [PATCH 38/38] Work on the docker test --- tests/test_multi_docker.py | 89 ++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/tests/test_multi_docker.py b/tests/test_multi_docker.py index fe3f7b8..c973682 100644 --- a/tests/test_multi_docker.py +++ b/tests/test_multi_docker.py @@ -13,8 +13,9 @@ def main(): sys.path.append(os.fspath(transcrypt_repo / 'tests')) test_dpath = ub.Path.appdir('transcrypt/tests/') - dpath1 = (test_dpath / 'container1').ensuredir() - dpath2 = (test_dpath / 'container2').ensuredir() + unshared_dpath1 = (test_dpath / 'unshared_container1').ensuredir() + unshared_dpath2 = (test_dpath / 'unshared_container2').ensuredir() + shared_dpath = (test_dpath / 'shared').ensuredir() transcrypt_repo @@ -25,41 +26,81 @@ def main(): image='ubuntu:20.04', volumes=[ (transcrypt_repo, '/transcrypt'), - (dpath1, '/custom'), + (unshared_dpath1, '/unshared'), + (shared_dpath, '/shared'), ]) container2 = OCIContainer( image='ubuntu:22.04', volumes=[ (transcrypt_repo, '/transcrypt'), - (dpath2, '/custom'), + (unshared_dpath2, '/unshared'), + (shared_dpath, '/shared'), ]) container1.start() container2.start() + try: - container1.call(['/bin/ls'], cwd='/custom') - container2.call(['/bin/ls'], cwd='/custom') + container1.call(['/bin/ls'], cwd='/unshared') + container2.call(['/bin/ls'], cwd='/unshared') + container1.call(['/bin/ls'], cwd='/shared') + container2.call(['/bin/ls'], cwd='/shared') - def setup_container(container): - container.call(shsplit('apt-get update -y')) - container.call(shsplit('apt-get install git -y')) - container.call(shsplit('git clone /transcrypt/.git'), cwd='/custom') - container.call(shsplit(f'git config --global user.email "{container.name}@test.com"')) - container.call(shsplit(f'git config --global user.name "{container.name}"')) - container.call(shsplit('git config --global init.defaultBranch "main"')) + def setup_container(container): + container.call(shsplit('apt-get update -y')) + container.call(shsplit('apt-get install git bsdmainutils xxd -y')) - container.call(shsplit('chmod +x /custom/transcrypt/transcrypt'), cwd='/custom') - container.call(shsplit('ls -al /custom/transcrypt'), cwd='/custom') - container.call(shsplit('ln -s /custom/transcrypt/transcrypt /usr/local/bin/transcrypt'), cwd='/custom') - container.call(shsplit('transcrypt --version'), cwd='/custom/transcrypt/example') + container.call(shsplit(f'git config --global user.email "{container.name}@test.com"')) + container.call(shsplit(f'git config --global user.name "{container.name}"')) + container.call(shsplit('git config --global init.defaultBranch "main"')) - container.call(shsplit('git status'), cwd='/custom/transcrypt') - container.call(shsplit('git pull'), cwd='/custom/transcrypt') - container.call(shsplit('bash end_to_end_example.sh'), cwd='/custom/transcrypt/example') + container.call(shsplit('ls /transcrypt')) + container.call(shsplit('mkdir -p /repos')) + container.call(shsplit('git clone /transcrypt/.git'), cwd='/repos') + container.call(shsplit('chmod +x /repos/transcrypt/transcrypt'), cwd='/repos') + container.call(shsplit('ls -al /repos/transcrypt'), cwd='/repos') + container.call(shsplit('ln -s /repos/transcrypt/transcrypt /usr/local/bin/transcrypt'), cwd='/repos') + container.call(shsplit('transcrypt --version'), cwd='/repos/transcrypt/example') - setup_container(container1) - setup_container(container2) + container.call(shsplit('git status'), cwd='/repos/transcrypt') + container.call(shsplit('git pull'), cwd='/repos/transcrypt') + container.call(shsplit('bash end_to_end_example.sh'), cwd='/repos/transcrypt/example') - container1.call(shsplit('git clone /transcrypt/.git'), cwd='/custom') - container2.call(shsplit('git clone /transcrypt/.git'), cwd='/custom') + container.call(shsplit('ls -al /root/tmp/transcrypt-demo/demo-repo-tc-end-to-end')) + container.call(shsplit('git clone /root/tmp/transcrypt-demo/demo-repo-tc-end-to-end/.git'), cwd='/unshared') + + # Ensure both containers have prereqs and run the end-to-end example on + # them to initialize a simple repo. + + # container = container1 + setup_container(container1) + setup_container(container2) + + # Setup one encrypted repo that the containers will both communicate with + container1.call(shsplit('git clone /root/tmp/transcrypt-demo/demo-repo-tc-end-to-end/.git /shared/encrypted'), cwd='/shared') + container1.call(shsplit('git clone /shared/encrypted/.git /shared/decrypted1'), cwd='/shared') + container2.call(shsplit('git clone /shared/encrypted/.git /shared/decrypted2'), cwd='/shared') + + # Decrypt in each one respectively + container1.call(shsplit("transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=pbkdf2 -y"), cwd='/shared/decrypted1') + # Strange. This seems to modify .transcrypt/config for some reason, and + # I don't know why or at least it says it does? It doesn't have a diff, + # but it is modified in the git status, but it's not even an encrypted + # file marked by gitattributes. + container1.call(shsplit("transcrypt -d"), cwd='/shared/decrypted1') + container1.call(shsplit("git status"), cwd='/shared/decrypted1') + container1.call(shsplit("git diff"), cwd='/shared/decrypted1') + + container2.call(shsplit("transcrypt -F -f -y"), cwd='/shared/decrypted2') + container2.call(shsplit("transcrypt -c aes-256-cbc -p 'correct horse battery staple' -md MD5 --kdf=pbkdf2 -y -bs 87c9f716d79a6a96bf84a5475b77c998"), cwd='/shared/decrypted2') + container2.call(shsplit("transcrypt -d"), cwd='/shared/decrypted2') + container2.call(shsplit("git status"), cwd='/shared/decrypted2') + container2.call(shsplit("git diff"), cwd='/shared/decrypted2') + container2.call(shsplit("cat safe/secret_file"), cwd='/shared/decrypted2') + + # container1.call(shsplit("cat safe/secret_file"), cwd='/shared/decrypted1') + + finally: + container1.stop() + container2.stop()