ciq-cherry-pick.py: Automatically cherry pick cve-bf commits

It now automatically cherry picks the Fixes: dependencies.

To accomodate this, CIQ_find_mainline_fixes was moved to ciq_helpers.
And an extra argument for upstream-ref was introduced, the default being
origin/kernel-mainline, as the dependencies are looked up there.

To simplify things and keep main cleaner, separate functions were used.

If one of the commits (the original and its dependencies) cannot be applied,
the return code will be 1. This is useful when ciq-cherry-pick.py is called
from other script.

This also removed redundant prints.

Signed-off-by: Roxana Nicolescu <rnicolescu@ciq.com>

Author: roxanan1996

check_kernel_commits.py

@@ -9,8 +9,7 @@
 from typing import Optional
 
 from ciq_helpers import (
-    CIQ_commit_exists_in_branch,
-    CIQ_extract_fixes_references_from_commit_body_lines,
+    CIQ_find_fixes_in_mainline,
     CIQ_get_commit_body,
     CIQ_hash_exists_in_ref,
     CIQ_run_git,
@@ -47,54 +46,6 @@ def hash_exists_in_mainline(repo, upstream_ref, hash_):
     return CIQ_hash_exists_in_ref(repo, upstream_ref, hash_)
 
 
-def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_):
-    """
-    Return unique commits in upstream_ref that have Fixes: <N chars of hash_> in their message, case-insensitive,
-    if they have not been committed in the pr_branch.
-    Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length.
-    Returns a list of tuples: (full_hash, display_string)
-    """
-    results = []
-
-    # Prepare hash prefixes from 12 down to 6
-    hash_prefixes = [hash_[:index] for index in range(12, 5, -1)]
-
-    # Get all commits with 'Fixes:' in the message
-    output = CIQ_run_git(
-        repo,
-        [
-            "log",
-            upstream_ref,
-            "--grep",
-            "Fixes:",
-            "-i",
-            "--format=%H %h %s (%an)%x0a%B%x00",
-        ],
-    ).strip()
-    if not output:
-        return []
-
-    # Each commit is separated by a NUL character and a newline
-    commits = output.split("\x00\x0a")
-    for commit in commits:
-        if not commit.strip():
-            continue
-
-        lines = commit.splitlines()
-        # The first line is the summary, the rest is the body
-        header = lines[0]
-        full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split())
-        fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:])
-        for fix in fixes:
-            for prefix in hash_prefixes:
-                if fix.lower().startswith(prefix.lower()):
-                    if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash):
-                        results.append((full_hash, display_string))
-                    break
-
-    return results
-
-
 def wrap_paragraph(text, width=80, initial_indent="", subsequent_indent=""):
     """Wrap a paragraph of text to the specified width and indentation."""
     wrapper = textwrap.TextWrapper(
@@ -238,7 +189,7 @@ def main():
                     )
                     out_lines.append("")  # blank line
                 continue
-            fixes = find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash)
+            fixes = CIQ_find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash)
             if fixes:
                 any_findings = True
 

ciq-cherry-pick.py

@@ -1,18 +1,23 @@
 import argparse
 import logging
 import os
+import re
 import subprocess
 
 import git
 
 from ciq_helpers import (
     CIQ_cherry_pick_commit_standardization,
     CIQ_commit_exists_in_current_branch,
+    CIQ_find_fixes_in_mainline_current_branch,
     CIQ_fixes_references,
+    CIQ_get_full_hash,
     CIQ_original_commit_author_to_tag_string,
+    CIQ_run_git,
 )
 
 MERGE_MSG = git.Repo(os.getcwd()).git_dir + "/MERGE_MSG"
+MERGE_MSG_BAK = f"{MERGE_MSG}.bak"
 
 
 def check_fixes(sha):
@@ -31,6 +36,100 @@ def check_fixes(sha):
             raise RuntimeError(f"The commit you want to cherry pick references a Fixes: {fix} but this is not here")
 
 
+def manage_commit_message(full_sha, ciq_tags, jira_ticket):
+    """
+    It standardize the commit message by including the ciq_tags, original
+    author and the original commit full sha.
+
+    Original message location: MERGE_MSG
+    Makes a copy of the original message in MERGE_MSG_BAK
+
+    The new standardized commit message is written to MERGE_MSG
+    """
+
+    subprocess.run(["cp", MERGE_MSG, MERGE_MSG_BAK], check=True)
+
+    # Make sure it's a deep copy because ciq_tags may be used for other cherry-picks
+    new_tags = [tag for tag in ciq_tags]
+
+    author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=full_sha)
+    if author is None:
+        raise RuntimeError(f"Could not find author of commit {full_sha}")
+
+    new_tags.append(author)
+    with open(MERGE_MSG, "r") as file:
+        original_msg = file.readlines()
+
+    new_msg = CIQ_cherry_pick_commit_standardization(original_msg, full_sha, jira=jira_ticket, tags=ciq_tags)
+
+    print(f"Cherry Pick New Message for {full_sha}")
+    print(f"\n Original Message located here: {MERGE_MSG_BAK}")
+
+    with open(MERGE_MSG, "w") as file:
+        file.writelines(new_msg)
+
+
+def cherry_pick(sha, ciq_tags, jira_ticket):
+    """
+    Cherry picks a commit and it adds the ciq standardized format
+    In case of error (cherry pick conflict):
+        - MERGE_MSG.bak contains the original commit message
+        - MERGE_MSG contains the standardized commit message
+        - Conflict has to be solved manualy
+
+    In case of success:
+        - the commit is cherry picked
+        - MERGE_MSG.bak is deleted
+        - You can still see MERGE_MSG for the original message
+    """
+
+    # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression
+    full_sha = CIQ_get_full_hash(repo=os.getcwd(), short_hash=sha)
+
+    check_fixes(sha=full_sha)
+
+    # Commit message is in MERGE_MSG
+    git_res = subprocess.run(["git", "cherry-pick", "-nsx", full_sha])
+    manage_commit_message(full_sha=full_sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket)
+
+    if git_res.returncode != 0:
+        error_str = (
+            f"[FAILED] git cherry-pick -nsx {full_sha}\n"
+            "Manually resolve conflict and include `upstream-diff` tag in commit message\n"
+            f"Subprocess Call: {git_res}"
+        )
+        raise RuntimeError(error_str)
+
+    CIQ_run_git(repo_path=os.getcwd(), args=["commit", "-F", MERGE_MSG])
+
+
+def cherry_pick_fixes(sha, ciq_tags, jira_ticket, upstream_ref):
+    """
+    It checks upstream_ref for commits that have this reference:
+    Fixes: <sha>. If any, these will also be cherry picked with the ciq
+    tag = cve-bf. If the tag was cve-pre, it stays the same.
+    """
+    fixes_in_mainline = CIQ_find_fixes_in_mainline_current_branch(os.getcwd(), upstream_ref, sha)
+
+    # Replace cve with cve-bf
+    # Leave cve-pre and cve-bf as they are
+    bf_ciq_tags = [re.sub(r"^cve ", "cve-bf ", s) for s in ciq_tags]
+    for full_hash, display_str in fixes_in_mainline:
+        print(f"Extra cherry picking {display_str}")
+        full_cherry_pick(sha=full_hash, ciq_tags=bf_ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref)
+
+
+def full_cherry_pick(sha, ciq_tags, jira_ticket, upstream_ref):
+    """
+    It cherry picks a commit from upstream-ref along with its Fixes: references.
+    """
+    # Cherry pick the commit
+    cherry_pick(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket)
+
+    # Cherry pick the fixed-by dependencies
+    cherry_pick_fixes(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref)
+
+
 if __name__ == "__main__":
     print("CIQ custom cherry picker")
     parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
@@ -44,59 +143,22 @@ def check_fixes(sha):
         "         cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n"
         "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n",
     )
+    parser.add_argument(
+        "--upstream-ref",
+        default="origin/kernel-mainline",
+        help="Reference to upstream mainline branch (default: origin/kernel-mainline)",
+    )
+
     args = parser.parse_args()
 
     logging.basicConfig(level=logging.INFO)
 
-    # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression
-    git_sha_res = subprocess.run(["git", "show", "--pretty=%H", "-s", args.sha], stdout=subprocess.PIPE)
-    if git_sha_res.returncode != 0:
-        print(f"[FAILED] git show --pretty=%H -s {args.sha}")
-        print("Subprocess Call:")
-        print(git_sha_res)
-        print("")
-    else:
-        args.sha = git_sha_res.stdout.decode("utf-8").strip()
-
     tags = []
     if args.ciq_tag is not None:
         tags = args.ciq_tag.split(",")
 
     try:
-        check_fixes(args.sha)
+        full_cherry_pick(sha=args.sha, ciq_tags=tags, jira_ticket=args.ticket, upstream_ref=args.upstream_ref)
     except Exception as e:
         print(e)
         exit(1)
-
-    author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha)
-    if author is None:
-        exit(1)
-
-    git_res = subprocess.run(["git", "cherry-pick", "-nsx", args.sha])
-    if git_res.returncode != 0:
-        print(f"[FAILED] git cherry-pick -nsx {args.sha}")
-        print("       Manually resolve conflict and include `upstream-diff` tag in commit message")
-        print("Subprocess Call:")
-        print(git_res)
-        print("")
-
-    print(os.getcwd())
-    subprocess.run(["cp", MERGE_MSG, f"{MERGE_MSG}.bak"])
-
-    tags.append(author)
-
-    with open(MERGE_MSG, "r") as file:
-        original_msg = file.readlines()
-
-    new_msg = CIQ_cherry_pick_commit_standardization(original_msg, args.sha, jira=args.ticket, tags=tags)
-
-    print(f"Cherry Pick New Message for {args.sha}")
-    for line in new_msg:
-        print(line.strip("\n"))
-    print(f"\n Original Message located here: {MERGE_MSG}.bak")
-
-    with open(MERGE_MSG, "w") as file:
-        file.writelines(new_msg)
-
-    if git_res.returncode == 0:
-        subprocess.run(["git", "commit", "-F", MERGE_MSG])

ciq_helpers.py

@@ -251,6 +251,60 @@ def CIQ_commit_exists_in_current_branch(repo, upstream_hash_):
     return CIQ_commit_exists_in_branch(repo, current_branch, full_upstream_hash)
 
 
+def CIQ_find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_):
+    """
+    Return unique commits in upstream_ref that have Fixes: <N chars of hash_> in their message, case-insensitive,
+    if they have not been committed in the pr_branch.
+    Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length.
+    Returns a list of tuples: (full_hash, display_string)
+    """
+    results = []
+
+    # Prepare hash prefixes from 12 down to 6
+    hash_prefixes = [hash_[:index] for index in range(12, 5, -1)]
+
+    # Get all commits with 'Fixes:' in the message
+    output = CIQ_run_git(
+        repo,
+        [
+            "log",
+            upstream_ref,
+            "--grep",
+            "Fixes:",
+            "-i",
+            "--format=%H %h %s (%an)%x0a%B%x00",
+        ],
+    ).strip()
+    if not output:
+        return []
+
+    # Each commit is separated by a NUL character and a newline
+    commits = output.split("\x00\x0a")
+    for commit in commits:
+        if not commit.strip():
+            continue
+
+        lines = commit.splitlines()
+        # The first line is the summary, the rest is the body
+        header = lines[0]
+        full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split())
+        fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:])
+        for fix in fixes:
+            for prefix in hash_prefixes:
+                if fix.lower().startswith(prefix.lower()):
+                    if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash):
+                        results.append((full_hash, display_string))
+                    break
+
+    return results
+
+
+def CIQ_find_fixes_in_mainline_current_branch(repo, upstream_ref, hash_):
+    current_branch = CIQ_get_current_branch(repo)
+
+    return CIQ_find_fixes_in_mainline(repo, current_branch, upstream_ref, hash_)
+
+
 def repo_init(repo):
     """Initialize a git repo object.