Merge branch 'hn/allow-bogus-oid-in-ref-tests'

The test helper for refs subsystem learned to write bogus and/or
nonexistent object name to refs to simulate error situations we
want to test Git in.

* hn/allow-bogus-oid-in-ref-tests:
  t1430: create valid symrefs using test-helper
  t1430: remove refs using test-tool
  refs: introduce REF_SKIP_REFNAME_VERIFICATION flag
  refs: introduce REF_SKIP_OID_VERIFICATION flag
  refs: update comment.
  test-ref-store: plug memory leak in cmd_delete_refs
  test-ref-store: parse symbolic flag constants
  test-ref-store: remove force-create argument for create-reflog

Author: gitster

refs.c

@@ -1083,9 +1083,10 @@ int ref_transaction_update(struct ref_transaction *transaction,
 {
 	assert(err);
 
-	if ((new_oid && !is_null_oid(new_oid)) ?
-	    check_refname_format(refname, REFNAME_ALLOW_ONELEVEL) :
-	    !refname_is_safe(refname)) {
+	if (!(flags & REF_SKIP_REFNAME_VERIFICATION) &&
+	    ((new_oid && !is_null_oid(new_oid)) ?
+		     check_refname_format(refname, REFNAME_ALLOW_ONELEVEL) :
+			   !refname_is_safe(refname))) {
 		strbuf_addf(err, _("refusing to update ref with bad name '%s'"),
 			    refname);
 		return -1;

refs.h

@@ -637,12 +637,24 @@ struct ref_transaction *ref_transaction_begin(struct strbuf *err);
  */
 #define REF_FORCE_CREATE_REFLOG (1 << 1)
 
+/*
+ * Blindly write an object_id. This is useful for testing data corruption
+ * scenarios.
+ */
+#define REF_SKIP_OID_VERIFICATION (1 << 10)
+
+/*
+ * Skip verifying refname. This is useful for testing data corruption scenarios.
+ */
+#define REF_SKIP_REFNAME_VERIFICATION (1 << 11)
+
 /*
  * Bitmask of all of the flags that are allowed to be passed in to
  * ref_transaction_update() and friends:
  */
-#define REF_TRANSACTION_UPDATE_ALLOWED_FLAGS \
-	(REF_NO_DEREF | REF_FORCE_CREATE_REFLOG)
+#define REF_TRANSACTION_UPDATE_ALLOWED_FLAGS                                  \
+	(REF_NO_DEREF | REF_FORCE_CREATE_REFLOG | REF_SKIP_OID_VERIFICATION | \
+	 REF_SKIP_REFNAME_VERIFICATION)
 
 /*
  * Add a reference update to transaction. `new_oid` is the value that

refs/files-backend.c

@@ -16,8 +16,7 @@
  * This backend uses the following flags in `ref_update::flags` for
  * internal bookkeeping purposes. Their numerical values must not
  * conflict with REF_NO_DEREF, REF_FORCE_CREATE_REFLOG, REF_HAVE_NEW,
- * REF_HAVE_OLD, or REF_IS_PRUNING, which are also stored in
- * `ref_update::flags`.
+ * or REF_HAVE_OLD, which are also stored in `ref_update::flags`.
  */
 
 /*
@@ -1354,7 +1353,8 @@ static int rename_tmp_log(struct files_ref_store *refs, const char *newrefname)
 }
 
 static int write_ref_to_lockfile(struct ref_lock *lock,
-				 const struct object_id *oid, struct strbuf *err);
+				 const struct object_id *oid,
+				 int skip_oid_verification, struct strbuf *err);
 static int commit_ref_update(struct files_ref_store *refs,
 			     struct ref_lock *lock,
 			     const struct object_id *oid, const char *logmsg,
@@ -1501,7 +1501,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store,
 	}
 	oidcpy(&lock->old_oid, &orig_oid);
 
-	if (write_ref_to_lockfile(lock, &orig_oid, &err) ||
+	if (write_ref_to_lockfile(lock, &orig_oid, 0, &err) ||
 	    commit_ref_update(refs, lock, &orig_oid, logmsg, &err)) {
 		error("unable to write current sha1 into %s: %s", newrefname, err.buf);
 		strbuf_release(&err);
@@ -1521,7 +1521,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store,
 
 	flag = log_all_ref_updates;
 	log_all_ref_updates = LOG_REFS_NONE;
-	if (write_ref_to_lockfile(lock, &orig_oid, &err) ||
+	if (write_ref_to_lockfile(lock, &orig_oid, 0, &err) ||
 	    commit_ref_update(refs, lock, &orig_oid, NULL, &err)) {
 		error("unable to write current sha1 into %s: %s", oldrefname, err.buf);
 		strbuf_release(&err);
@@ -1756,26 +1756,31 @@ static int files_log_ref_write(struct files_ref_store *refs,
  * errors, rollback the lockfile, fill in *err and return -1.
  */
 static int write_ref_to_lockfile(struct ref_lock *lock,
-				 const struct object_id *oid, struct strbuf *err)
+				 const struct object_id *oid,
+				 int skip_oid_verification, struct strbuf *err)
 {
 	static char term = '\n';
 	struct object *o;
 	int fd;
 
-	o = parse_object(the_repository, oid);
-	if (!o) {
-		strbuf_addf(err,
-			    "trying to write ref '%s' with nonexistent object %s",
-			    lock->ref_name, oid_to_hex(oid));
-		unlock_ref(lock);
-		return -1;
-	}
-	if (o->type != OBJ_COMMIT && is_branch(lock->ref_name)) {
-		strbuf_addf(err,
-			    "trying to write non-commit object %s to branch '%s'",
-			    oid_to_hex(oid), lock->ref_name);
-		unlock_ref(lock);
-		return -1;
+	if (!skip_oid_verification) {
+		o = parse_object(the_repository, oid);
+		if (!o) {
+			strbuf_addf(
+				err,
+				"trying to write ref '%s' with nonexistent object %s",
+				lock->ref_name, oid_to_hex(oid));
+			unlock_ref(lock);
+			return -1;
+		}
+		if (o->type != OBJ_COMMIT && is_branch(lock->ref_name)) {
+			strbuf_addf(
+				err,
+				"trying to write non-commit object %s to branch '%s'",
+				oid_to_hex(oid), lock->ref_name);
+			unlock_ref(lock);
+			return -1;
+		}
 	}
 	fd = get_lock_file_fd(&lock->lk);
 	if (write_in_full(fd, oid_to_hex(oid), the_hash_algo->hexsz) < 0 ||
@@ -2189,7 +2194,7 @@ static int files_reflog_iterator_advance(struct ref_iterator *ref_iterator)
 }
 
 static int files_reflog_iterator_peel(struct ref_iterator *ref_iterator,
-				   struct object_id *peeled)
+				      struct object_id *peeled)
 {
 	BUG("ref_iterator_peel() called for reflog_iterator");
 }
@@ -2575,8 +2580,10 @@ static int lock_ref_for_update(struct files_ref_store *refs,
 			 * The reference already has the desired
 			 * value, so we don't need to write it.
 			 */
-		} else if (write_ref_to_lockfile(lock, &update->new_oid,
-						 err)) {
+		} else if (write_ref_to_lockfile(
+				   lock, &update->new_oid,
+				   update->flags & REF_SKIP_OID_VERIFICATION,
+				   err)) {
 			char *write_err = strbuf_detach(err, NULL);
 
 			/*

t/helper/test-ref-store.c

@@ -5,16 +5,59 @@
 #include "object-store.h"
 #include "repository.h"
 
+struct flag_definition {
+	const char *name;
+	uint64_t mask;
+};
+
+#define FLAG_DEF(x)     \
+	{               \
+#x, (x) \
+	}
+
+static unsigned int parse_flags(const char *str, struct flag_definition *defs)
+{
+	struct string_list masks = STRING_LIST_INIT_DUP;
+	int i = 0;
+	unsigned int result = 0;
+
+	if (!strcmp(str, "0"))
+		return 0;
+
+	string_list_split(&masks, str, ',', 64);
+	for (; i < masks.nr; i++) {
+		const char *name = masks.items[i].string;
+		struct flag_definition *def = defs;
+		int found = 0;
+		while (def->name) {
+			if (!strcmp(def->name, name)) {
+				result |= def->mask;
+				found = 1;
+				break;
+			}
+			def++;
+		}
+		if (!found)
+			die("unknown flag \"%s\"", name);
+	}
+
+	string_list_clear(&masks, 0);
+	return result;
+}
+
+static struct flag_definition empty_flags[] = { { NULL, 0 } };
+
 static const char *notnull(const char *arg, const char *name)
 {
 	if (!arg)
 		die("%s required", name);
 	return arg;
 }
 
-static unsigned int arg_flags(const char *arg, const char *name)
+static unsigned int arg_flags(const char *arg, const char *name,
+			      struct flag_definition *defs)
 {
-	return atoi(notnull(arg, name));
+	return parse_flags(notnull(arg, name), defs);
 }
 
 static const char **get_store(const char **argv, struct ref_store **refs)
@@ -64,10 +107,13 @@ static const char **get_store(const char **argv, struct ref_store **refs)
 	return argv + 1;
 }
 
+static struct flag_definition pack_flags[] = { FLAG_DEF(PACK_REFS_PRUNE),
+					       FLAG_DEF(PACK_REFS_ALL),
+					       { NULL, 0 } };
 
 static int cmd_pack_refs(struct ref_store *refs, const char **argv)
 {
-	unsigned int flags = arg_flags(*argv++, "flags");
+	unsigned int flags = arg_flags(*argv++, "flags", pack_flags);
 
 	return refs_pack_refs(refs, flags);
 }
@@ -81,16 +127,27 @@ static int cmd_create_symref(struct ref_store *refs, const char **argv)
 	return refs_create_symref(refs, refname, target, logmsg);
 }
 
+static struct flag_definition transaction_flags[] = {
+	FLAG_DEF(REF_NO_DEREF),
+	FLAG_DEF(REF_FORCE_CREATE_REFLOG),
+	FLAG_DEF(REF_SKIP_OID_VERIFICATION),
+	FLAG_DEF(REF_SKIP_REFNAME_VERIFICATION),
+	{ NULL, 0 }
+};
+
 static int cmd_delete_refs(struct ref_store *refs, const char **argv)
 {
-	unsigned int flags = arg_flags(*argv++, "flags");
+	unsigned int flags = arg_flags(*argv++, "flags", transaction_flags);
 	const char *msg = *argv++;
 	struct string_list refnames = STRING_LIST_INIT_NODUP;
+	int result;
 
 	while (*argv)
 		string_list_append(&refnames, *argv++);
 
-	return refs_delete_refs(refs, msg, &refnames, flags);
+	result = refs_delete_refs(refs, msg, &refnames, flags);
+	string_list_clear(&refnames, 0);
+	return result;
 }
 
 static int cmd_rename_ref(struct ref_store *refs, const char **argv)
@@ -120,7 +177,7 @@ static int cmd_resolve_ref(struct ref_store *refs, const char **argv)
 {
 	struct object_id oid = *null_oid();
 	const char *refname = notnull(*argv++, "refname");
-	int resolve_flags = arg_flags(*argv++, "resolve-flags");
+	int resolve_flags = arg_flags(*argv++, "resolve-flags", empty_flags);
 	int flags;
 	const char *ref;
 	int ignore_errno;
@@ -208,7 +265,7 @@ static int cmd_delete_ref(struct ref_store *refs, const char **argv)
 	const char *msg = notnull(*argv++, "msg");
 	const char *refname = notnull(*argv++, "refname");
 	const char *sha1_buf = notnull(*argv++, "old-sha1");
-	unsigned int flags = arg_flags(*argv++, "flags");
+	unsigned int flags = arg_flags(*argv++, "flags", transaction_flags);
 	struct object_id old_oid;
 
 	if (get_oid_hex(sha1_buf, &old_oid))
@@ -223,7 +280,7 @@ static int cmd_update_ref(struct ref_store *refs, const char **argv)
 	const char *refname = notnull(*argv++, "refname");
 	const char *new_sha1_buf = notnull(*argv++, "new-sha1");
 	const char *old_sha1_buf = notnull(*argv++, "old-sha1");
-	unsigned int flags = arg_flags(*argv++, "flags");
+	unsigned int flags = arg_flags(*argv++, "flags", transaction_flags);
 	struct object_id old_oid;
 	struct object_id new_oid;
 

t/t1006-cat-file.sh

@@ -452,9 +452,8 @@ test_expect_success 'the --allow-unknown-type option does not consider replaceme
 	# Create it manually, as "git replace" will die on bogus
 	# types.
 	head=$(git rev-parse --verify HEAD) &&
-	test_when_finished "rm -rf .git/refs/replace" &&
-	mkdir -p .git/refs/replace &&
-	echo $head >.git/refs/replace/$bogus_short_sha1 &&
+	test_when_finished "test-tool ref-store main delete-refs 0 msg refs/replace/$bogus_short_sha1" &&
+	test-tool ref-store main update-ref msg "refs/replace/$bogus_short_sha1" $head $ZERO_OID REF_SKIP_OID_VERIFICATION &&
 
 	cat >expect <<-EOF &&
 	commit

t/t1405-main-ref-store.sh

@@ -17,8 +17,7 @@ test_expect_success 'setup' '
 test_expect_success REFFILES 'pack_refs(PACK_REFS_ALL | PACK_REFS_PRUNE)' '
 	N=`find .git/refs -type f | wc -l` &&
 	test "$N" != 0 &&
-	ALL_OR_PRUNE_FLAG=3 &&
-	$RUN pack-refs ${ALL_OR_PRUNE_FLAG} &&
+	$RUN pack-refs PACK_REFS_PRUNE,PACK_REFS_ALL &&
 	N=`find .git/refs -type f` &&
 	test -z "$N"
 '
@@ -35,8 +34,7 @@ test_expect_success 'delete_refs(FOO, refs/tags/new-tag)' '
 	git rev-parse FOO -- &&
 	git rev-parse refs/tags/new-tag -- &&
 	m=$(git rev-parse main) &&
-	REF_NO_DEREF=1 &&
-	$RUN delete-refs $REF_NO_DEREF nothing FOO refs/tags/new-tag &&
+	$RUN delete-refs REF_NO_DEREF nothing FOO refs/tags/new-tag &&
 	test_must_fail git rev-parse --symbolic-full-name FOO &&
 	test_must_fail git rev-parse FOO -- &&
 	test_must_fail git rev-parse refs/tags/new-tag --