diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2fb31c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ + +# Ruff stuff: +.ruff_cache/ + +venv/ +env/ +.venv/ +.env/ + +*.egg-info/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8d390b3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.14.0 + hooks: + # Run the linter. + - id: ruff-check + args: [ --fix ] + # Run the formatter. + - id: ruff-format + +exclude: | + (?x)^( + jira_pr_check.py | + release_config.py | + rolling-release-update.py | + run_interdiff.py | + )$ diff --git a/check_kernel_commits.py b/check_kernel_commits.py index 0d72f94..ee5450c 100644 --- a/check_kernel_commits.py +++ b/check_kernel_commits.py @@ -1,150 +1,159 @@ #!/usr/bin/env python3 import argparse -import subprocess +import os import re +import subprocess import sys import textwrap -import os from typing import Optional -def run_git(repo, args): - """Run a git command in the given repository and return its output as a string.""" - result = subprocess.run(['git', '-C', repo] + args, text=True, capture_output=True, check=False) - if result.returncode != 0: - raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") - return result.stdout +from ciq_helpers import ( + CIQ_commit_exists_in_branch, + CIQ_extract_fixes_references_from_commit_body_lines, + CIQ_find_fixes_in_mainline, + CIQ_get_commit_body, + CIQ_hash_exists_in_ref, + CIQ_run_git, +) + def ref_exists(repo, ref): """Return True if the given ref exists in the repository, False otherwise.""" try: - run_git(repo, ['rev-parse', '--verify', '--quiet', ref]) + CIQ_run_git(repo, ["rev-parse", "--verify", "--quiet", ref]) return True except RuntimeError: return False + def get_pr_commits(repo, pr_branch, base_branch): """Get a list of commit SHAs that are in the PR branch but not in the base branch.""" - output = run_git(repo, ['rev-list', f'{base_branch}..{pr_branch}']) + output = CIQ_run_git(repo, ["rev-list", f"{base_branch}..{pr_branch}"]) return output.strip().splitlines() -def get_commit_message(repo, sha): - """Get the commit message for a given commit SHA.""" - return run_git(repo, ['log', '-n', '1', '--format=%B', sha]) def get_short_hash_and_subject(repo, sha): """Get the abbreviated commit hash and subject for a given commit SHA.""" - output = run_git(repo, ['log', '-n', '1', '--format=%h%x00%s', sha]).strip() - short_hash, subject = output.split('\x00', 1) + output = CIQ_run_git(repo, ["log", "-n", "1", "--format=%h%x00%s", sha]).strip() + short_hash, subject = output.split("\x00", 1) return short_hash, subject + def hash_exists_in_mainline(repo, upstream_ref, hash_): """ Return True if hash_ is reachable from upstream_ref (i.e., is an ancestor of it). """ - try: - run_git(repo, ['merge-base', '--is-ancestor', hash_, upstream_ref]) - return True - except RuntimeError: - return False + + return CIQ_hash_exists_in_ref(repo, upstream_ref, hash_) + def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): """ - Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive. + Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive, + if they have not been commited in the pr_branch. Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length. Returns a list of tuples: (full_hash, display_string) """ results = [] + + # Prepare hash prefixes from 12 down to 6 + hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] + # Get all commits with 'Fixes:' in the message - output = run_git(repo, [ - 'log', upstream_ref, '--grep', 'Fixes:', '-i', '--format=%H %h %s (%an)%x0a%B%x00' - ]).strip() + output = CIQ_run_git( + repo, + [ + "log", + upstream_ref, + "--grep", + "Fixes:", + "-i", + "--format=%H %h %s (%an)%x0a%B%x00", + ], + ).strip() if not output: return [] + # Each commit is separated by a NUL character and a newline - commits = output.split('\x00\x0a') - # Prepare hash prefixes from 12 down to 6 - hash_prefixes = [hash_[:l] for l in range(12, 5, -1)] + commits = output.split("\x00\x0a") for commit in commits: if not commit.strip(): continue - # The first line is the summary, the rest is the body + lines = commit.splitlines() - if not lines: - continue + # The first line is the summary, the rest is the body header = lines[0] - full_hash = header.split()[0] - # Search for Fixes: lines in the commit message - for line in lines[1:]: - m = re.match(r'^\s*Fixes:\s*([0-9a-fA-F]{6,40})', line, re.IGNORECASE) - if m: - for prefix in hash_prefixes: - if m.group(1).lower().startswith(prefix.lower()): - if not commit_exists_in_branch(repo, pr_branch, full_hash): - results.append((full_hash, ' '.join(header.split()[1:]))) - break - else: - continue - return results + full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split()) + fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:]) + for fix in fixes: + for prefix in hash_prefixes: + if fix.lower().startswith(prefix.lower()): + if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash): + results.append((full_hash, display_string)) + break -def commit_exists_in_branch(repo, pr_branch, upstream_hash_): - """ - Return True if upstream_hash_ has been backported and it exists in the - pr branch - """ - output = run_git(repo, ['log', pr_branch, '--grep', 'commit ' + upstream_hash_]) - if not output: - return False + return results - return True -def wrap_paragraph(text, width=80, initial_indent='', subsequent_indent=''): +def wrap_paragraph(text, width=80, initial_indent="", subsequent_indent=""): """Wrap a paragraph of text to the specified width and indentation.""" - wrapper = textwrap.TextWrapper(width=width, - initial_indent=initial_indent, - subsequent_indent=subsequent_indent, - break_long_words=False, - break_on_hyphens=False) + wrapper = textwrap.TextWrapper( + width=width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent, + break_long_words=False, + break_on_hyphens=False, + ) return wrapper.fill(text) + def extract_cve_from_message(msg): """Extract CVE reference from commit message. Returns CVE ID or None. Only matches 'cve CVE-2025-12345', ignores 'cve-bf' and 'cve-pre' variants.""" - match = re.search(r'(? tuple[bool, Optional[str]]: """ Run the cve_search script from the vulns repo. Returns (success, output_message). """ - cve_search_path = os.path.join(vulns_repo, 'scripts', 'cve_search') + cve_search_path = os.path.join(vulns_repo, "scripts", "cve_search") if not os.path.exists(cve_search_path): raise RuntimeError(f"cve_search script not found at {cve_search_path}") env = os.environ.copy() - env['CVEKERNELTREE'] = kernel_repo + env["CVEKERNELTREE"] = kernel_repo - result = subprocess.run([cve_search_path, query], - text=True, - capture_output=True, - check=False, - env=env) + result = subprocess.run([cve_search_path, query], text=True, capture_output=True, check=False, env=env) # cve_search outputs results to stdout return result.returncode == 0, result.stdout.strip() + def main(): parser = argparse.ArgumentParser(description="Check upstream references and Fixes: tags in PR branch commits.") parser.add_argument("--repo", help="Path to the git repo", required=True) parser.add_argument("--pr_branch", help="Name of the PR branch", required=True) parser.add_argument("--base_branch", help="Name of the base branch", required=True) - parser.add_argument("--markdown", action='store_true', help="Output in Markdown, suitable for GitHub PR comments") - parser.add_argument("--upstream-ref", default="origin/kernel-mainline", help="Reference to upstream mainline branch (default: origin/kernel-mainline)") - parser.add_argument("--check-cves", action='store_true', help="Check that CVE references in commit messages match upstream commit hashes") - parser.add_argument("--vulns-dir", default="../vulns", help="Path to the kernel vulnerabilities repo (default: ../vulns)") + parser.add_argument("--markdown", action="store_true", help="Output in Markdown, suitable for GitHub PR comments") + parser.add_argument( + "--upstream-ref", + default="origin/kernel-mainline", + help="Reference to upstream mainline branch (default: origin/kernel-mainline)", + ) + parser.add_argument( + "--check-cves", + action="store_true", + help="Check that CVE references in commit messages match upstream commit hashes", + ) + parser.add_argument( + "--vulns-dir", default="../vulns", help="Path to the kernel vulnerabilities repo (default: ../vulns)" + ) args = parser.parse_args() upstream_ref = args.upstream_ref @@ -158,17 +167,16 @@ def main(): if os.path.exists(vulns_repo): # Repository exists, update it with git pull try: - run_git(vulns_repo, ['pull']) + CIQ_run_git(vulns_repo, ["pull"]) except RuntimeError as e: print(f"WARNING: Failed to update vulns repo: {e}") print("Continuing with existing repository...") else: # Repository doesn't exist, clone it try: - result = subprocess.run(['git', 'clone', vulns_repo_url, vulns_repo], - text=True, - capture_output=True, - check=False) + result = subprocess.run( + ["git", "clone", vulns_repo_url, vulns_repo], text=True, capture_output=True, check=False + ) if result.returncode != 0: print(f"ERROR: Failed to clone vulns repo: {result.stderr}") sys.exit(1) @@ -178,9 +186,11 @@ def main(): # Validate that all required refs exist before continuing missing_refs = [] - for refname, refval in [('upstream reference', upstream_ref), - ('PR branch', args.pr_branch), - ('base branch', args.base_branch)]: + for refname, refval in [ + ("upstream reference", upstream_ref), + ("PR branch", args.pr_branch), + ("base branch", args.base_branch), + ]: if not ref_exists(args.repo, refval): missing_refs.append((refname, refval)) if missing_refs: @@ -203,8 +213,8 @@ def main(): for sha in reversed(pr_commits): # oldest first short_hash, subject = get_short_hash_and_subject(args.repo, sha) pr_commit_desc = f"{short_hash} ({subject})" - msg = get_commit_message(args.repo, sha) - upstream_hashes = re.findall(r'^commit\s+([0-9a-fA-F]{40})', msg, re.MULTILINE) + msg = CIQ_get_commit_body(args.repo, sha) + upstream_hashes = re.findall(r"^commit\s+([0-9a-fA-F]{40})", msg, re.MULTILINE) for uhash in upstream_hashes: short_uhash = uhash[:12] # Ensure the referenced commit in the PR actually exists in the upstream ref. @@ -218,15 +228,18 @@ def main(): ) else: prefix = "[NOTFOUND] " - header = (f"{prefix}PR commit {pr_commit_desc} references upstream commit " - f"{short_uhash}, which does not exist in kernel-mainline.") + header = ( + f"{prefix}PR commit {pr_commit_desc} references upstream commit " + f"{short_uhash}, which does not exist in kernel-mainline." + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) # spaces for '[NOTFOUND] ' + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) # spaces for '[NOTFOUND] ' ) out_lines.append("") # blank line continue - fixes = find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash) + fixes = CIQ_find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash) if fixes: any_findings = True @@ -238,7 +251,7 @@ def main(): success, cve_output = run_cve_search(vulns_repo, args.repo, fix_hash) if success: # Parse the CVE from the result - match = re.search(r'(CVE-\d{4}-\d+)\s+is assigned to git id', cve_output) + match = re.search(r"(CVE-\d{4}-\d+)\s+is assigned to git id", cve_output) if match: bugfix_cve = match.group(1) fix_cves[fix_hash] = bugfix_cve @@ -265,15 +278,18 @@ def main(): ) else: prefix = "[FIXES] " - header = (f"{prefix}PR commit {pr_commit_desc} references upstream commit " - f"{short_uhash}, which has Fixes tags:") + header = ( + f"{prefix}PR commit {pr_commit_desc} references upstream commit " + f"{short_uhash}, which has Fixes tags:" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) # spaces for '[FIXES] ' + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) # spaces for '[FIXES] ' ) out_lines.append("") # blank line after 'Fixes tags:' for line in fixes_text.splitlines(): - out_lines.append(' ' + line) + out_lines.append(" " + line) out_lines.append("") # blank line # Check CVE if enabled @@ -285,8 +301,9 @@ def main(): success, cve_output = run_cve_search(vulns_repo, args.repo, uhash) if success: # Parse the output to get the CVE from the result - # Expected format: "CVE-2024-35962 is assigned to git id 65acf6e0501ac8880a4f73980d01b5d27648b956" - match = re.search(r'(CVE-\d{4}-\d+)\s+is assigned to git id', cve_output) + # Expected format: "CVE-2024-35962 is assigned to git id + # 65acf6e0501ac8880a4f73980d01b5d27648b956" + match = re.search(r"(CVE-\d{4}-\d+)\s+is assigned to git id", cve_output) if match: found_cve = match.group(1) @@ -301,11 +318,14 @@ def main(): ) else: prefix = "[CVE-MISMATCH] " - header = (f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " - f"upstream commit {short_uhash} is associated with {found_cve}") + header = ( + f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " + f"upstream commit {short_uhash} is associated with {found_cve}" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) ) out_lines.append("") # blank line else: @@ -318,11 +338,14 @@ def main(): ) else: prefix = "[CVE-MISSING] " - header = (f"{prefix}PR commit {pr_commit_desc} does not reference a CVE but " - f"upstream commit {short_uhash} is associated with {found_cve}") + header = ( + f"{prefix}PR commit {pr_commit_desc} does not reference a CVE but " + f"upstream commit {short_uhash} is associated with {found_cve}" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) ) out_lines.append("") # blank line else: @@ -337,11 +360,14 @@ def main(): ) else: prefix = "[CVE-NOTFOUND] " - header = (f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " - f"upstream commit {short_uhash} has no CVE assigned") + header = ( + f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " + f"upstream commit {short_uhash} has no CVE assigned" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) ) out_lines.append("") # blank line except (subprocess.SubprocessError, OSError) as e: @@ -355,26 +381,25 @@ def main(): ) else: prefix = "[CVE-ERROR] " - header = (f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " - f"failed to verify: {e}") + header = f"{prefix}PR commit {pr_commit_desc} references {cve_id} but failed to verify: {e}" out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph(header, width=80, initial_indent="", subsequent_indent=" " * len(prefix)) ) out_lines.append("") # blank line if any_findings: if args.markdown: print("## :mag: Upstream Linux Kernel Commit Check\n") - print('\n'.join(out_lines)) + print("\n".join(out_lines)) print("*This is an automated message from the kernel commit checker workflow.*") else: - print('\n'.join(out_lines)) + print("\n".join(out_lines)) else: if args.markdown: print("> ✅ **All referenced commits exist upstream and have no Fixes: tags.**") else: print("All referenced commits exist upstream and have no Fixes: tags.") + if __name__ == "__main__": main() diff --git a/ciq-cherry-pick.py b/ciq-cherry-pick.py index bfa9c3a..6cf6fd9 100644 --- a/ciq-cherry-pick.py +++ b/ciq-cherry-pick.py @@ -1,44 +1,50 @@ import argparse +import logging import os +import re import subprocess + import git -from ciq_helpers import CIQ_cherry_pick_commit_standardization -from ciq_helpers import CIQ_original_commit_author_to_tag_string -# from ciq_helpers import * -MERGE_MSG = git.Repo(os.getcwd()).git_dir + '/MERGE_MSG' +from ciq_helpers import ( + CIQ_cherry_pick_commit_standardization, + CIQ_commit_exists_in_current_branch, + CIQ_find_fixes_in_mainline_current_branch, + CIQ_fixes_references, + CIQ_get_full_hash, + CIQ_original_commit_author_to_tag_string, +) -if __name__ == '__main__': - print("CIQ custom cherry picker") - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--sha', help='Target SHA1 to cherry-pick') - parser.add_argument('--ticket', help='Ticket associated to cherry-pick work, comma separated list is supported.') - parser.add_argument('--ciq-tag', help="Tags for commit message <-optional modifier> .\n" - "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n" - " cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n" - " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" - "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n") - args = parser.parse_args() +MERGE_MSG = git.Repo(os.getcwd()).git_dir + "/MERGE_MSG" + + +def check_fixes(sha): + """ + Checks if commit has "Fixes:" references and if so, it checks if the + commit(s) that it tries to fix are part of the current branch + """ + + fixes = CIQ_fixes_references(repo_path=".", sha=sha) + if len(fixes) == 0: + logging.warning("The commit you try to cherry pick has no Fixes: reference; review it carefully") + return + + for fix in fixes: + if not CIQ_commit_exists_in_current_branch(".", fix): + raise RuntimeError(f"The commit you want to cherry pick references a Fixes {fix}: but this is not here") + +def cherry_pick(sha, ciq_tags, jira_ticket): # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression - git_sha_res = subprocess.run(['git', 'show', '--pretty=%H', '-s', args.sha], stdout=subprocess.PIPE) - if git_sha_res.returncode != 0: - print(f"[FAILED] git show --pretty=%H -s {args.sha}") - print("Subprocess Call:") - print(git_sha_res) - print("") - else: - args.sha = git_sha_res.stdout.decode('utf-8').strip() + full_sha = CIQ_get_full_hash(".", sha) - tags = [] - if args.ciq_tag is not None: - tags = args.ciq_tag.split(',') + check_fixes(full_sha) - author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha) - if author is None: + author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=full_sha) + if author is None: # TODO raise Exception maybe exit(1) - git_res = subprocess.run(['git', 'cherry-pick', '-nsx', args.sha]) + git_res = subprocess.run(["git", "cherry-pick", "-nsx", full_sha]) # Move it into a separate method if git_res.returncode != 0: print(f"[FAILED] git cherry-pick -nsx {args.sha}") print(" Manually resolve conflict and include `upstream-diff` tag in commit message") @@ -47,22 +53,70 @@ print("") print(os.getcwd()) - subprocess.run(['cp', MERGE_MSG, f'{MERGE_MSG}.bak']) + subprocess.run(["cp", MERGE_MSG, f"{MERGE_MSG}.bak"]) tags.append(author) with open(MERGE_MSG, "r") as file: original_msg = file.readlines() - new_msg = CIQ_cherry_pick_commit_standardization(original_msg, args.sha, jira=args.ticket, tags=tags) + new_msg = CIQ_cherry_pick_commit_standardization(original_msg, full_sha, jira=jira_ticket, tags=ciq_tags) print(f"Cherry Pick New Message for {args.sha}") for line in new_msg: - print(line.strip('\n')) + print(line.strip("\n")) print(f"\n Original Message located here: {MERGE_MSG}.bak") with open(MERGE_MSG, "w") as file: file.writelines(new_msg) if git_res.returncode == 0: - subprocess.run(['git', 'commit', '-F', MERGE_MSG]) + subprocess.run(["git", "commit", "-F", MERGE_MSG]) + + +def cherry_pick_fixes(sha, ciq_tags, jira_ticket, upstream_ref): + fixes_in_mainline = CIQ_find_fixes_in_mainline_current_branch(".", upstream_ref, sha) + + # Replace cve with cve-bf + # Leave cve-pre and cve-bf as they are + ciq_tags = [re.sub(r"^cve ", "cve-bf ", s) for s in ciq_tags] + for full_hash, display_str in fixes_in_mainline: + print(f"Extra cherry picking {display_str}") + full_cherry_pick(sha=full_hash, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref) + + +def full_cherry_pick(sha, ciq_tags, jira_ticket, upstream_ref): + # Cherry pick the commit + cherry_pick(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket) + + # Cherry pick the fixed-by dependencies + cherry_pick_fixes(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref) + + +if __name__ == "__main__": + print("CIQ custom cherry picker") + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("--sha", help="Target SHA1 to cherry-pick") + parser.add_argument("--ticket", help="Ticket associated to cherry-pick work, comma separated list is supported.") + parser.add_argument( + "--ciq-tag", + help="Tags for commit message <-optional modifier> .\n" + "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n" + " cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n" + " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" + "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n", + ) + + parser.add_argument( + "--upstream-ref", + default="origin/kernel-mainline", + help="Reference to upstream mainline branch (default: origin/kernel-mainline)", + ) + + args = parser.parse_args() + + tags = [] + if args.ciq_tag is not None: + tags = args.ciq_tag.split(",") + + full_cherry_pick(sha=args.sha, ciq_tags=tags, jira_ticket=args.ticket, upstream_ref=args.upstream_ref) diff --git a/ciq_helpers.py b/ciq_helpers.py index 7284c64..b8d62c2 100644 --- a/ciq_helpers.py +++ b/ciq_helpers.py @@ -3,11 +3,12 @@ # CIQ Kernel Tools function library -import git import os import re import subprocess +import git + def process_full_commit_message(commit): """Process the full git commit message specific to the CIQ Kernel Tools. @@ -84,9 +85,7 @@ def get_backport_commit_data(repo, branch, common_ancestor, allow_duplicates=Fal for line in lines: if len(commit) > 0 and line.startswith(b"commit "): - upstream_commit, cves, tickets, upstream_subject, repo_commit = ( - process_full_commit_message(commit) - ) + upstream_commit, cves, tickets, upstream_subject, repo_commit = process_full_commit_message(commit) if upstream_commit in upstream_commits: print(f"WARNING: {upstream_commit} already in upstream_commits") if not allow_duplicates: @@ -104,9 +103,7 @@ def get_backport_commit_data(repo, branch, common_ancestor, allow_duplicates=Fal return upstream_commits, True -def CIQ_cherry_pick_commit_standardization( - lines, commit, tags=None, jira="", optional_msg="" -): +def CIQ_cherry_pick_commit_standardization(lines, commit, tags=None, jira="", optional_msg=""): """Standardize CIQ the cherry-pick commit message. Parameters: lines: Original SHAS commit message. @@ -159,13 +156,154 @@ def CIQ_original_commit_author_to_tag_string(repo_path, sha): Return: String for Tag """ - git_auth_res = subprocess.run(['git', 'show', '--pretty="%aN <%aE>"', '--no-patch', sha], stderr=subprocess.PIPE, - stdout=subprocess.PIPE, cwd=repo_path) + git_auth_res = subprocess.run( + ["git", "show", '--pretty="%aN <%aE>"', "--no-patch", sha], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + cwd=repo_path, + ) if git_auth_res.returncode != 0: print(f"[FAILED] git show --pretty='%aN <%aE>' --no-patch {sha}") print(f"[FAILED][STDERR:{git_auth_res.returncode}] {git_auth_res.stderr.decode('utf-8')}") return None - return "commit-author " + git_auth_res.stdout.decode('utf-8').replace('\"', '').strip() + return "commit-author " + git_auth_res.stdout.decode("utf-8").replace('"', "").strip() + + +def CIQ_run_git(repo_path, args): + """ + Run a git command in the given repository and return its output as a string. + """ + result = subprocess.run(["git", "-C", repo_path] + args, text=True, capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") + + return result.stdout + + +def CIQ_get_commit_body(repo_path, sha): + return CIQ_run_git(repo_path, ["show", "-s", sha, "--format=%B"]) + + +def CIQ_extract_fixes_references_from_commit_body_lines(lines): + fixes = [] + for line in lines: + m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE) + if not m: + continue + + fixes.append(m.group(1)) + + return fixes + + +def CIQ_fixes_references(repo_path, sha): + """ + If commit message of sha contains lines like + Fixes: , this returns a list of , otherwise an empty list + """ + + commit_body = CIQ_get_commit_body(repo_path, sha) + return CIQ_extract_fixes_references_from_commit_body_lines(lines=commit_body.splitlines()) + + +def CIQ_get_full_hash(repo, short_hash): + return CIQ_run_git(repo, ["show", "-s", "--pretty=%H", short_hash]).strip() + + +def CIQ_get_current_branch(repo): + return CIQ_run_git(repo, ["branch", "--show-current"]).strip() + + +def CIQ_hash_exists_in_ref(repo, pr_ref, hash_): + """ + Return True is hash_ is reachable from pr_branch + """ + + try: + CIQ_run_git(repo, ["merge-base", "--is-ancestor", hash_, pr_ref]) + return True + except RuntimeError: + return False + + +# TODO think of a better name +def CIQ_commit_exists_in_branch(repo, pr_branch, upstream_hash_): + """ + Return True if upstream_hash_ has been backported and it exists in the pr branch + """ + + # First check if the commit has been backported by CIQ + output = CIQ_run_git(repo, ["log", pr_branch, "--grep", "commit " + upstream_hash_]) + if output: + return True + + # If it was not backported by CIQ, maybe it came from upstream as it is + return CIQ_hash_exists_in_ref(repo, pr_branch, upstream_hash_) + + +def CIQ_commit_exists_in_current_branch(repo, upstream_hash_): + """ + Return True if upstream_hash_ has been backported and it exists in the current branch + """ + + current_branch = CIQ_get_current_branch(repo) + full_upstream_hash = CIQ_get_full_hash(repo, upstream_hash_) + + return CIQ_commit_exists_in_branch(repo, current_branch, full_upstream_hash) + + +def CIQ_find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): + """ + Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive, + if they have not been commited in the pr_branch. + Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length. + Returns a list of tuples: (full_hash, display_string) + """ + results = [] + + # Prepare hash prefixes from 12 down to 6 + hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] + + # Get all commits with 'Fixes:' in the message + output = CIQ_run_git( + repo, + [ + "log", + upstream_ref, + "--grep", + "Fixes:", + "-i", + "--format=%H %h %s (%an)%x0a%B%x00", + ], + ).strip() + if not output: + return [] + + # Each commit is separated by a NUL character and a newline + commits = output.split("\x00\x0a") + for commit in commits: + if not commit.strip(): + continue + + lines = commit.splitlines() + # The first line is the summary, the rest is the body + header = lines[0] + full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split()) + fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:]) + for fix in fixes: + for prefix in hash_prefixes: + if fix.lower().startswith(prefix.lower()): + if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash): + results.append((full_hash, display_string)) + break + + return results + + +def CIQ_find_fixes_in_mainline_current_branch(repo, upstream_ref, hash_): + current_branch = CIQ_get_current_branch(repo) + + return CIQ_find_fixes_in_mainline(repo, current_branch, upstream_ref, hash_) def repo_init(repo): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dc6678b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +# This is not an installable project at the moment. +# This is used to control dependencies and liter configuration +[project] +name = "kernel-src-tree-tools" +requires-python = ">=3.10" +readme = "README.md" +version = "0.0.1" +dependencies = [ +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "ruff", + ] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +# Add the `line-too-long` rule to the enforced rule set. +extend-select = ["E501"] + +[tool.setuptools] +# suppress error: Multiple top-level packages +py-modules = []