From 472dcefb839ef1b713cdb9874e0477eaf9ab5775 Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Mon, 3 Nov 2025 13:33:42 +0100 Subject: [PATCH 1/7] packaging: setup tooling framework Add support for linting tools and dependencies management. The file needed are: - pyproject.toml. This is used for managing dependencies and configuration into a single file. At the moment, it is not written to be able to install this project as a proper package, but it can be changed in the future. - .pre-commit-config.yaml. This will run Ruff's linter and formatter via pre-commit. At the moment, all *py are excluded from pre-commit formatting. Signed-off-by: Roxana Nicolescu --- .pre-commit-config.yaml | 21 +++++++++++++++++++++ pyproject.toml | 26 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..572b9db --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.14.0 + hooks: + # Run the linter. + - id: ruff-check + args: [ --fix ] + # Run the formatter. + - id: ruff-format + +exclude: | + (?x)^( + ciq-cherry-pick.py | + check_kernel_commits.py | + ciq_helpers.py | + jira_pr_check.py | + release_config.py | + rolling-release-update.py | + run_interdiff.py | + )$ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dc6678b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +# This is not an installable project at the moment. +# This is used to control dependencies and liter configuration +[project] +name = "kernel-src-tree-tools" +requires-python = ">=3.10" +readme = "README.md" +version = "0.0.1" +dependencies = [ +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "ruff", + ] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +# Add the `line-too-long` rule to the enforced rule set. +extend-select = ["E501"] + +[tool.setuptools] +# suppress error: Multiple top-level packages +py-modules = [] From 77d0b5b4e283ec822011989fdd884fa874b9aadb Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Mon, 3 Nov 2025 13:35:24 +0100 Subject: [PATCH 2/7] Add .gitignore Signed-off-by: Roxana Nicolescu --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2fb31c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ + +# Ruff stuff: +.ruff_cache/ + +venv/ +env/ +.venv/ +.env/ + +*.egg-info/ From 2746af69545990ef77bd7cc337d1c32385ab1536 Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Sun, 9 Nov 2025 15:11:42 +0100 Subject: [PATCH 3/7] check_kernel_commits.py: Ruff formatting This was done automatically by ruff format. Default configuration is used. Variable l was replaced by index due to #E741. It is not recommened to use variables named 'I', 'O' or 'l' because they are hard to read. Signed-off-by: Roxana Nicolescu --- .pre-commit-config.yaml | 1 - check_kernel_commits.py | 182 ++++++++++++++++++++++++---------------- 2 files changed, 108 insertions(+), 75 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 572b9db..a6c40de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,6 @@ repos: exclude: | (?x)^( ciq-cherry-pick.py | - check_kernel_commits.py | ciq_helpers.py | jira_pr_check.py | release_config.py | diff --git a/check_kernel_commits.py b/check_kernel_commits.py index 0d72f94..4cd8198 100644 --- a/check_kernel_commits.py +++ b/check_kernel_commits.py @@ -1,53 +1,60 @@ #!/usr/bin/env python3 import argparse -import subprocess +import os import re +import subprocess import sys import textwrap -import os from typing import Optional + def run_git(repo, args): """Run a git command in the given repository and return its output as a string.""" - result = subprocess.run(['git', '-C', repo] + args, text=True, capture_output=True, check=False) + result = subprocess.run(["git", "-C", repo] + args, text=True, capture_output=True, check=False) if result.returncode != 0: raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") return result.stdout + def ref_exists(repo, ref): """Return True if the given ref exists in the repository, False otherwise.""" try: - run_git(repo, ['rev-parse', '--verify', '--quiet', ref]) + run_git(repo, ["rev-parse", "--verify", "--quiet", ref]) return True except RuntimeError: return False + def get_pr_commits(repo, pr_branch, base_branch): """Get a list of commit SHAs that are in the PR branch but not in the base branch.""" - output = run_git(repo, ['rev-list', f'{base_branch}..{pr_branch}']) + output = run_git(repo, ["rev-list", f"{base_branch}..{pr_branch}"]) return output.strip().splitlines() + def get_commit_message(repo, sha): """Get the commit message for a given commit SHA.""" - return run_git(repo, ['log', '-n', '1', '--format=%B', sha]) + return run_git(repo, ["log", "-n", "1", "--format=%B", sha]) + def get_short_hash_and_subject(repo, sha): """Get the abbreviated commit hash and subject for a given commit SHA.""" - output = run_git(repo, ['log', '-n', '1', '--format=%h%x00%s', sha]).strip() - short_hash, subject = output.split('\x00', 1) + output = run_git(repo, ["log", "-n", "1", "--format=%h%x00%s", sha]).strip() + short_hash, subject = output.split("\x00", 1) return short_hash, subject + def hash_exists_in_mainline(repo, upstream_ref, hash_): """ Return True if hash_ is reachable from upstream_ref (i.e., is an ancestor of it). """ try: - run_git(repo, ['merge-base', '--is-ancestor', hash_, upstream_ref]) + run_git(repo, ["merge-base", "--is-ancestor", hash_, upstream_ref]) return True except RuntimeError: return False + def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): """ Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive. @@ -56,15 +63,13 @@ def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): """ results = [] # Get all commits with 'Fixes:' in the message - output = run_git(repo, [ - 'log', upstream_ref, '--grep', 'Fixes:', '-i', '--format=%H %h %s (%an)%x0a%B%x00' - ]).strip() + output = run_git(repo, ["log", upstream_ref, "--grep", "Fixes:", "-i", "--format=%H %h %s (%an)%x0a%B%x00"]).strip() if not output: return [] # Each commit is separated by a NUL character and a newline - commits = output.split('\x00\x0a') + commits = output.split("\x00\x0a") # Prepare hash prefixes from 12 down to 6 - hash_prefixes = [hash_[:l] for l in range(12, 5, -1)] + hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] for commit in commits: if not commit.strip(): continue @@ -76,75 +81,88 @@ def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): full_hash = header.split()[0] # Search for Fixes: lines in the commit message for line in lines[1:]: - m = re.match(r'^\s*Fixes:\s*([0-9a-fA-F]{6,40})', line, re.IGNORECASE) + m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE) if m: for prefix in hash_prefixes: if m.group(1).lower().startswith(prefix.lower()): if not commit_exists_in_branch(repo, pr_branch, full_hash): - results.append((full_hash, ' '.join(header.split()[1:]))) + results.append((full_hash, " ".join(header.split()[1:]))) break else: continue return results + def commit_exists_in_branch(repo, pr_branch, upstream_hash_): """ Return True if upstream_hash_ has been backported and it exists in the pr branch """ - output = run_git(repo, ['log', pr_branch, '--grep', 'commit ' + upstream_hash_]) + output = run_git(repo, ["log", pr_branch, "--grep", "commit " + upstream_hash_]) if not output: return False return True -def wrap_paragraph(text, width=80, initial_indent='', subsequent_indent=''): + +def wrap_paragraph(text, width=80, initial_indent="", subsequent_indent=""): """Wrap a paragraph of text to the specified width and indentation.""" - wrapper = textwrap.TextWrapper(width=width, - initial_indent=initial_indent, - subsequent_indent=subsequent_indent, - break_long_words=False, - break_on_hyphens=False) + wrapper = textwrap.TextWrapper( + width=width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent, + break_long_words=False, + break_on_hyphens=False, + ) return wrapper.fill(text) + def extract_cve_from_message(msg): """Extract CVE reference from commit message. Returns CVE ID or None. Only matches 'cve CVE-2025-12345', ignores 'cve-bf' and 'cve-pre' variants.""" - match = re.search(r'(? tuple[bool, Optional[str]]: """ Run the cve_search script from the vulns repo. Returns (success, output_message). """ - cve_search_path = os.path.join(vulns_repo, 'scripts', 'cve_search') + cve_search_path = os.path.join(vulns_repo, "scripts", "cve_search") if not os.path.exists(cve_search_path): raise RuntimeError(f"cve_search script not found at {cve_search_path}") env = os.environ.copy() - env['CVEKERNELTREE'] = kernel_repo + env["CVEKERNELTREE"] = kernel_repo - result = subprocess.run([cve_search_path, query], - text=True, - capture_output=True, - check=False, - env=env) + result = subprocess.run([cve_search_path, query], text=True, capture_output=True, check=False, env=env) # cve_search outputs results to stdout return result.returncode == 0, result.stdout.strip() + def main(): parser = argparse.ArgumentParser(description="Check upstream references and Fixes: tags in PR branch commits.") parser.add_argument("--repo", help="Path to the git repo", required=True) parser.add_argument("--pr_branch", help="Name of the PR branch", required=True) parser.add_argument("--base_branch", help="Name of the base branch", required=True) - parser.add_argument("--markdown", action='store_true', help="Output in Markdown, suitable for GitHub PR comments") - parser.add_argument("--upstream-ref", default="origin/kernel-mainline", help="Reference to upstream mainline branch (default: origin/kernel-mainline)") - parser.add_argument("--check-cves", action='store_true', help="Check that CVE references in commit messages match upstream commit hashes") - parser.add_argument("--vulns-dir", default="../vulns", help="Path to the kernel vulnerabilities repo (default: ../vulns)") + parser.add_argument("--markdown", action="store_true", help="Output in Markdown, suitable for GitHub PR comments") + parser.add_argument( + "--upstream-ref", + default="origin/kernel-mainline", + help="Reference to upstream mainline branch (default: origin/kernel-mainline)", + ) + parser.add_argument( + "--check-cves", + action="store_true", + help="Check that CVE references in commit messages match upstream commit hashes", + ) + parser.add_argument( + "--vulns-dir", default="../vulns", help="Path to the kernel vulnerabilities repo (default: ../vulns)" + ) args = parser.parse_args() upstream_ref = args.upstream_ref @@ -158,17 +176,16 @@ def main(): if os.path.exists(vulns_repo): # Repository exists, update it with git pull try: - run_git(vulns_repo, ['pull']) + run_git(vulns_repo, ["pull"]) except RuntimeError as e: print(f"WARNING: Failed to update vulns repo: {e}") print("Continuing with existing repository...") else: # Repository doesn't exist, clone it try: - result = subprocess.run(['git', 'clone', vulns_repo_url, vulns_repo], - text=True, - capture_output=True, - check=False) + result = subprocess.run( + ["git", "clone", vulns_repo_url, vulns_repo], text=True, capture_output=True, check=False + ) if result.returncode != 0: print(f"ERROR: Failed to clone vulns repo: {result.stderr}") sys.exit(1) @@ -178,9 +195,11 @@ def main(): # Validate that all required refs exist before continuing missing_refs = [] - for refname, refval in [('upstream reference', upstream_ref), - ('PR branch', args.pr_branch), - ('base branch', args.base_branch)]: + for refname, refval in [ + ("upstream reference", upstream_ref), + ("PR branch", args.pr_branch), + ("base branch", args.base_branch), + ]: if not ref_exists(args.repo, refval): missing_refs.append((refname, refval)) if missing_refs: @@ -204,7 +223,7 @@ def main(): short_hash, subject = get_short_hash_and_subject(args.repo, sha) pr_commit_desc = f"{short_hash} ({subject})" msg = get_commit_message(args.repo, sha) - upstream_hashes = re.findall(r'^commit\s+([0-9a-fA-F]{40})', msg, re.MULTILINE) + upstream_hashes = re.findall(r"^commit\s+([0-9a-fA-F]{40})", msg, re.MULTILINE) for uhash in upstream_hashes: short_uhash = uhash[:12] # Ensure the referenced commit in the PR actually exists in the upstream ref. @@ -218,11 +237,14 @@ def main(): ) else: prefix = "[NOTFOUND] " - header = (f"{prefix}PR commit {pr_commit_desc} references upstream commit " - f"{short_uhash}, which does not exist in kernel-mainline.") + header = ( + f"{prefix}PR commit {pr_commit_desc} references upstream commit " + f"{short_uhash}, which does not exist in kernel-mainline." + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) # spaces for '[NOTFOUND] ' + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) # spaces for '[NOTFOUND] ' ) out_lines.append("") # blank line continue @@ -238,7 +260,7 @@ def main(): success, cve_output = run_cve_search(vulns_repo, args.repo, fix_hash) if success: # Parse the CVE from the result - match = re.search(r'(CVE-\d{4}-\d+)\s+is assigned to git id', cve_output) + match = re.search(r"(CVE-\d{4}-\d+)\s+is assigned to git id", cve_output) if match: bugfix_cve = match.group(1) fix_cves[fix_hash] = bugfix_cve @@ -265,15 +287,18 @@ def main(): ) else: prefix = "[FIXES] " - header = (f"{prefix}PR commit {pr_commit_desc} references upstream commit " - f"{short_uhash}, which has Fixes tags:") + header = ( + f"{prefix}PR commit {pr_commit_desc} references upstream commit " + f"{short_uhash}, which has Fixes tags:" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) # spaces for '[FIXES] ' + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) # spaces for '[FIXES] ' ) out_lines.append("") # blank line after 'Fixes tags:' for line in fixes_text.splitlines(): - out_lines.append(' ' + line) + out_lines.append(" " + line) out_lines.append("") # blank line # Check CVE if enabled @@ -285,8 +310,9 @@ def main(): success, cve_output = run_cve_search(vulns_repo, args.repo, uhash) if success: # Parse the output to get the CVE from the result - # Expected format: "CVE-2024-35962 is assigned to git id 65acf6e0501ac8880a4f73980d01b5d27648b956" - match = re.search(r'(CVE-\d{4}-\d+)\s+is assigned to git id', cve_output) + # Expected format: "CVE-2024-35962 is assigned to git id + # 65acf6e0501ac8880a4f73980d01b5d27648b956" + match = re.search(r"(CVE-\d{4}-\d+)\s+is assigned to git id", cve_output) if match: found_cve = match.group(1) @@ -301,11 +327,14 @@ def main(): ) else: prefix = "[CVE-MISMATCH] " - header = (f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " - f"upstream commit {short_uhash} is associated with {found_cve}") + header = ( + f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " + f"upstream commit {short_uhash} is associated with {found_cve}" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) ) out_lines.append("") # blank line else: @@ -318,11 +347,14 @@ def main(): ) else: prefix = "[CVE-MISSING] " - header = (f"{prefix}PR commit {pr_commit_desc} does not reference a CVE but " - f"upstream commit {short_uhash} is associated with {found_cve}") + header = ( + f"{prefix}PR commit {pr_commit_desc} does not reference a CVE but " + f"upstream commit {short_uhash} is associated with {found_cve}" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) ) out_lines.append("") # blank line else: @@ -337,11 +369,14 @@ def main(): ) else: prefix = "[CVE-NOTFOUND] " - header = (f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " - f"upstream commit {short_uhash} has no CVE assigned") + header = ( + f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " + f"upstream commit {short_uhash} has no CVE assigned" + ) out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph( + header, width=80, initial_indent="", subsequent_indent=" " * len(prefix) + ) ) out_lines.append("") # blank line except (subprocess.SubprocessError, OSError) as e: @@ -355,26 +390,25 @@ def main(): ) else: prefix = "[CVE-ERROR] " - header = (f"{prefix}PR commit {pr_commit_desc} references {cve_id} but " - f"failed to verify: {e}") + header = f"{prefix}PR commit {pr_commit_desc} references {cve_id} but failed to verify: {e}" out_lines.append( - wrap_paragraph(header, width=80, initial_indent='', - subsequent_indent=' ' * len(prefix)) + wrap_paragraph(header, width=80, initial_indent="", subsequent_indent=" " * len(prefix)) ) out_lines.append("") # blank line if any_findings: if args.markdown: print("## :mag: Upstream Linux Kernel Commit Check\n") - print('\n'.join(out_lines)) + print("\n".join(out_lines)) print("*This is an automated message from the kernel commit checker workflow.*") else: - print('\n'.join(out_lines)) + print("\n".join(out_lines)) else: if args.markdown: print("> ✅ **All referenced commits exist upstream and have no Fixes: tags.**") else: print("All referenced commits exist upstream and have no Fixes: tags.") + if __name__ == "__main__": main() From f4486be1b9c78ff01780dcf616a053ead4b754ff Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Sun, 9 Nov 2025 15:30:42 +0100 Subject: [PATCH 4/7] ciq-cherry-pick.py: Ruff formatting This was done automatically by ruff format. Default configuration is used. Signed-off-by: Roxana Nicolescu --- .pre-commit-config.yaml | 1 - ciq-cherry-pick.py | 41 +++++++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6c40de..5174a2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,6 @@ repos: exclude: | (?x)^( - ciq-cherry-pick.py | ciq_helpers.py | jira_pr_check.py | release_config.py | diff --git a/ciq-cherry-pick.py b/ciq-cherry-pick.py index bfa9c3a..9dded99 100644 --- a/ciq-cherry-pick.py +++ b/ciq-cherry-pick.py @@ -1,44 +1,49 @@ import argparse import os import subprocess + import git -from ciq_helpers import CIQ_cherry_pick_commit_standardization -from ciq_helpers import CIQ_original_commit_author_to_tag_string + +from ciq_helpers import CIQ_cherry_pick_commit_standardization, CIQ_original_commit_author_to_tag_string + # from ciq_helpers import * -MERGE_MSG = git.Repo(os.getcwd()).git_dir + '/MERGE_MSG' +MERGE_MSG = git.Repo(os.getcwd()).git_dir + "/MERGE_MSG" -if __name__ == '__main__': +if __name__ == "__main__": print("CIQ custom cherry picker") parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--sha', help='Target SHA1 to cherry-pick') - parser.add_argument('--ticket', help='Ticket associated to cherry-pick work, comma separated list is supported.') - parser.add_argument('--ciq-tag', help="Tags for commit message <-optional modifier> .\n" - "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n" - " cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n" - " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" - "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n") + parser.add_argument("--sha", help="Target SHA1 to cherry-pick") + parser.add_argument("--ticket", help="Ticket associated to cherry-pick work, comma separated list is supported.") + parser.add_argument( + "--ciq-tag", + help="Tags for commit message <-optional modifier> .\n" + "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n" + " cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n" + " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" + "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n", + ) args = parser.parse_args() # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression - git_sha_res = subprocess.run(['git', 'show', '--pretty=%H', '-s', args.sha], stdout=subprocess.PIPE) + git_sha_res = subprocess.run(["git", "show", "--pretty=%H", "-s", args.sha], stdout=subprocess.PIPE) if git_sha_res.returncode != 0: print(f"[FAILED] git show --pretty=%H -s {args.sha}") print("Subprocess Call:") print(git_sha_res) print("") else: - args.sha = git_sha_res.stdout.decode('utf-8').strip() + args.sha = git_sha_res.stdout.decode("utf-8").strip() tags = [] if args.ciq_tag is not None: - tags = args.ciq_tag.split(',') + tags = args.ciq_tag.split(",") author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha) if author is None: exit(1) - git_res = subprocess.run(['git', 'cherry-pick', '-nsx', args.sha]) + git_res = subprocess.run(["git", "cherry-pick", "-nsx", args.sha]) if git_res.returncode != 0: print(f"[FAILED] git cherry-pick -nsx {args.sha}") print(" Manually resolve conflict and include `upstream-diff` tag in commit message") @@ -47,7 +52,7 @@ print("") print(os.getcwd()) - subprocess.run(['cp', MERGE_MSG, f'{MERGE_MSG}.bak']) + subprocess.run(["cp", MERGE_MSG, f"{MERGE_MSG}.bak"]) tags.append(author) @@ -58,11 +63,11 @@ print(f"Cherry Pick New Message for {args.sha}") for line in new_msg: - print(line.strip('\n')) + print(line.strip("\n")) print(f"\n Original Message located here: {MERGE_MSG}.bak") with open(MERGE_MSG, "w") as file: file.writelines(new_msg) if git_res.returncode == 0: - subprocess.run(['git', 'commit', '-F', MERGE_MSG]) + subprocess.run(["git", "commit", "-F", MERGE_MSG]) From a19fa7ad10d8b435e5b606ec386ada44accbb118 Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Sun, 9 Nov 2025 16:44:57 +0100 Subject: [PATCH 5/7] ciq_helper.py: Ruff formatting This was done automatically by ruff format. Default configuration is used. Signed-off-by: Roxana Nicolescu --- .pre-commit-config.yaml | 1 - ciq_helpers.py | 21 +++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5174a2c..8d390b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,6 @@ repos: exclude: | (?x)^( - ciq_helpers.py | jira_pr_check.py | release_config.py | rolling-release-update.py | diff --git a/ciq_helpers.py b/ciq_helpers.py index 7284c64..e97f11d 100644 --- a/ciq_helpers.py +++ b/ciq_helpers.py @@ -3,11 +3,12 @@ # CIQ Kernel Tools function library -import git import os import re import subprocess +import git + def process_full_commit_message(commit): """Process the full git commit message specific to the CIQ Kernel Tools. @@ -84,9 +85,7 @@ def get_backport_commit_data(repo, branch, common_ancestor, allow_duplicates=Fal for line in lines: if len(commit) > 0 and line.startswith(b"commit "): - upstream_commit, cves, tickets, upstream_subject, repo_commit = ( - process_full_commit_message(commit) - ) + upstream_commit, cves, tickets, upstream_subject, repo_commit = process_full_commit_message(commit) if upstream_commit in upstream_commits: print(f"WARNING: {upstream_commit} already in upstream_commits") if not allow_duplicates: @@ -104,9 +103,7 @@ def get_backport_commit_data(repo, branch, common_ancestor, allow_duplicates=Fal return upstream_commits, True -def CIQ_cherry_pick_commit_standardization( - lines, commit, tags=None, jira="", optional_msg="" -): +def CIQ_cherry_pick_commit_standardization(lines, commit, tags=None, jira="", optional_msg=""): """Standardize CIQ the cherry-pick commit message. Parameters: lines: Original SHAS commit message. @@ -159,13 +156,17 @@ def CIQ_original_commit_author_to_tag_string(repo_path, sha): Return: String for Tag """ - git_auth_res = subprocess.run(['git', 'show', '--pretty="%aN <%aE>"', '--no-patch', sha], stderr=subprocess.PIPE, - stdout=subprocess.PIPE, cwd=repo_path) + git_auth_res = subprocess.run( + ["git", "show", '--pretty="%aN <%aE>"', "--no-patch", sha], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + cwd=repo_path, + ) if git_auth_res.returncode != 0: print(f"[FAILED] git show --pretty='%aN <%aE>' --no-patch {sha}") print(f"[FAILED][STDERR:{git_auth_res.returncode}] {git_auth_res.stderr.decode('utf-8')}") return None - return "commit-author " + git_auth_res.stdout.decode('utf-8').replace('\"', '').strip() + return "commit-author " + git_auth_res.stdout.decode("utf-8").replace('"', "").strip() def repo_init(repo): From 80703a69d1b45f5bf8521cc5717483a549a54796 Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Mon, 10 Nov 2025 10:10:23 +0100 Subject: [PATCH 6/7] ciq-cherry-pick.py: Cherry pick commit only if the Fixes: references are commited If the commit that needs to be cherry picked has "Fixes:" references in the commit body, there is now a check in pace that verify if those commits are present in the current branch. At the moment, the scrips returns an Exception because the developer must check why the commit has to be cherry picked for a bug fix or cve fix if the actual commit that introduced the bug/cve was not commited. If the commit does not reference any Fixes:, an warning is shown to make the developer aware that they have to double check if it makes sense to cherry pick this commit. The script continues as this can be reviewed after. This is common in the linux kernel community. Not all fixes have a Fixes: reference. Checking if a commit is part of the branch has now improved. It checks if either the commit was backported by our team, or if the commit came from upstream. Note: The implementation reuses some of the logic in the check_kernel_commits.py. Those have been moved to ciq_helper.py. This commit address the small refactor in check_kernel_commits.py as well. Signed-off-by: Roxana Nicolescu --- check_kernel_commits.py | 96 +++++++++++++--------------- ciq-cherry-pick.py | 29 ++++++++- ciq_helpers.py | 137 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 56 deletions(-) diff --git a/check_kernel_commits.py b/check_kernel_commits.py index 4cd8198..da273d0 100644 --- a/check_kernel_commits.py +++ b/check_kernel_commits.py @@ -8,19 +8,19 @@ import textwrap from typing import Optional - -def run_git(repo, args): - """Run a git command in the given repository and return its output as a string.""" - result = subprocess.run(["git", "-C", repo] + args, text=True, capture_output=True, check=False) - if result.returncode != 0: - raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") - return result.stdout +from ciq_helpers import ( + CIQ_commit_exists_in_branch, + CIQ_extract_fixes_references_from_commit_body_lines, + CIQ_get_commit_body, + CIQ_hash_exists_in_ref, + CIQ_run_git, +) def ref_exists(repo, ref): """Return True if the given ref exists in the repository, False otherwise.""" try: - run_git(repo, ["rev-parse", "--verify", "--quiet", ref]) + CIQ_run_git(repo, ["rev-parse", "--verify", "--quiet", ref]) return True except RuntimeError: return False @@ -28,18 +28,13 @@ def ref_exists(repo, ref): def get_pr_commits(repo, pr_branch, base_branch): """Get a list of commit SHAs that are in the PR branch but not in the base branch.""" - output = run_git(repo, ["rev-list", f"{base_branch}..{pr_branch}"]) + output = CIQ_run_git(repo, ["rev-list", f"{base_branch}..{pr_branch}"]) return output.strip().splitlines() -def get_commit_message(repo, sha): - """Get the commit message for a given commit SHA.""" - return run_git(repo, ["log", "-n", "1", "--format=%B", sha]) - - def get_short_hash_and_subject(repo, sha): """Get the abbreviated commit hash and subject for a given commit SHA.""" - output = run_git(repo, ["log", "-n", "1", "--format=%h%x00%s", sha]).strip() + output = CIQ_run_git(repo, ["log", "-n", "1", "--format=%h%x00%s", sha]).strip() short_hash, subject = output.split("\x00", 1) return short_hash, subject @@ -48,61 +43,56 @@ def hash_exists_in_mainline(repo, upstream_ref, hash_): """ Return True if hash_ is reachable from upstream_ref (i.e., is an ancestor of it). """ - try: - run_git(repo, ["merge-base", "--is-ancestor", hash_, upstream_ref]) - return True - except RuntimeError: - return False + + return CIQ_hash_exists_in_ref(repo, upstream_ref, hash_) def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): """ - Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive. + Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive, + if they have not been commited in the pr_branch. Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length. Returns a list of tuples: (full_hash, display_string) """ results = [] + + # Prepare hash prefixes from 12 down to 6 + hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] + # Get all commits with 'Fixes:' in the message - output = run_git(repo, ["log", upstream_ref, "--grep", "Fixes:", "-i", "--format=%H %h %s (%an)%x0a%B%x00"]).strip() + output = CIQ_run_git( + repo, + [ + "log", + upstream_ref, + "--grep", + "Fixes:", + "-i", + "--format=%H %h %s (%an)%x0a%B%x00", + ], + ).strip() if not output: return [] + # Each commit is separated by a NUL character and a newline commits = output.split("\x00\x0a") - # Prepare hash prefixes from 12 down to 6 - hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] for commit in commits: if not commit.strip(): continue - # The first line is the summary, the rest is the body + lines = commit.splitlines() - if not lines: - continue + # The first line is the summary, the rest is the body header = lines[0] - full_hash = header.split()[0] - # Search for Fixes: lines in the commit message - for line in lines[1:]: - m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE) - if m: - for prefix in hash_prefixes: - if m.group(1).lower().startswith(prefix.lower()): - if not commit_exists_in_branch(repo, pr_branch, full_hash): - results.append((full_hash, " ".join(header.split()[1:]))) - break - else: - continue - return results + full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split()) + fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:]) + for fix in fixes: + for prefix in hash_prefixes: + if fix.lower().startswith(prefix.lower()): + if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash): + results.append((full_hash, display_string)) + break - -def commit_exists_in_branch(repo, pr_branch, upstream_hash_): - """ - Return True if upstream_hash_ has been backported and it exists in the - pr branch - """ - output = run_git(repo, ["log", pr_branch, "--grep", "commit " + upstream_hash_]) - if not output: - return False - - return True + return results def wrap_paragraph(text, width=80, initial_indent="", subsequent_indent=""): @@ -176,7 +166,7 @@ def main(): if os.path.exists(vulns_repo): # Repository exists, update it with git pull try: - run_git(vulns_repo, ["pull"]) + CIQ_run_git(vulns_repo, ["pull"]) except RuntimeError as e: print(f"WARNING: Failed to update vulns repo: {e}") print("Continuing with existing repository...") @@ -222,7 +212,7 @@ def main(): for sha in reversed(pr_commits): # oldest first short_hash, subject = get_short_hash_and_subject(args.repo, sha) pr_commit_desc = f"{short_hash} ({subject})" - msg = get_commit_message(args.repo, sha) + msg = CIQ_get_commit_body(args.repo, sha) upstream_hashes = re.findall(r"^commit\s+([0-9a-fA-F]{40})", msg, re.MULTILINE) for uhash in upstream_hashes: short_uhash = uhash[:12] diff --git a/ciq-cherry-pick.py b/ciq-cherry-pick.py index 9dded99..5358bbc 100644 --- a/ciq-cherry-pick.py +++ b/ciq-cherry-pick.py @@ -1,15 +1,36 @@ import argparse +import logging import os import subprocess import git -from ciq_helpers import CIQ_cherry_pick_commit_standardization, CIQ_original_commit_author_to_tag_string - -# from ciq_helpers import * +from ciq_helpers import ( + CIQ_cherry_pick_commit_standardization, + CIQ_commit_exists_in_current_branch, + CIQ_fixes_references, + CIQ_original_commit_author_to_tag_string, +) MERGE_MSG = git.Repo(os.getcwd()).git_dir + "/MERGE_MSG" + +def check_fixes(sha): + """ + Checks if commit has "Fixes:" references and if so, it checks if the + commit(s) that it tries to fix are part of the current branch + """ + + fixes = CIQ_fixes_references(repo_path=".", sha=sha) + if len(fixes) == 0: + logging.warning("The commit you try to cherry pick has no Fixes: reference; review it carefully") + return + + for fix in fixes: + if not CIQ_commit_exists_in_current_branch(".", fix): + raise RuntimeError(f"The commit you want to cherry pick references a Fixes {fix}: but this is not here") + + if __name__ == "__main__": print("CIQ custom cherry picker") parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) @@ -39,6 +60,8 @@ if args.ciq_tag is not None: tags = args.ciq_tag.split(",") + check_fixes(args.sha) + author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha) if author is None: exit(1) diff --git a/ciq_helpers.py b/ciq_helpers.py index e97f11d..b8d62c2 100644 --- a/ciq_helpers.py +++ b/ciq_helpers.py @@ -169,6 +169,143 @@ def CIQ_original_commit_author_to_tag_string(repo_path, sha): return "commit-author " + git_auth_res.stdout.decode("utf-8").replace('"', "").strip() +def CIQ_run_git(repo_path, args): + """ + Run a git command in the given repository and return its output as a string. + """ + result = subprocess.run(["git", "-C", repo_path] + args, text=True, capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") + + return result.stdout + + +def CIQ_get_commit_body(repo_path, sha): + return CIQ_run_git(repo_path, ["show", "-s", sha, "--format=%B"]) + + +def CIQ_extract_fixes_references_from_commit_body_lines(lines): + fixes = [] + for line in lines: + m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE) + if not m: + continue + + fixes.append(m.group(1)) + + return fixes + + +def CIQ_fixes_references(repo_path, sha): + """ + If commit message of sha contains lines like + Fixes: , this returns a list of , otherwise an empty list + """ + + commit_body = CIQ_get_commit_body(repo_path, sha) + return CIQ_extract_fixes_references_from_commit_body_lines(lines=commit_body.splitlines()) + + +def CIQ_get_full_hash(repo, short_hash): + return CIQ_run_git(repo, ["show", "-s", "--pretty=%H", short_hash]).strip() + + +def CIQ_get_current_branch(repo): + return CIQ_run_git(repo, ["branch", "--show-current"]).strip() + + +def CIQ_hash_exists_in_ref(repo, pr_ref, hash_): + """ + Return True is hash_ is reachable from pr_branch + """ + + try: + CIQ_run_git(repo, ["merge-base", "--is-ancestor", hash_, pr_ref]) + return True + except RuntimeError: + return False + + +# TODO think of a better name +def CIQ_commit_exists_in_branch(repo, pr_branch, upstream_hash_): + """ + Return True if upstream_hash_ has been backported and it exists in the pr branch + """ + + # First check if the commit has been backported by CIQ + output = CIQ_run_git(repo, ["log", pr_branch, "--grep", "commit " + upstream_hash_]) + if output: + return True + + # If it was not backported by CIQ, maybe it came from upstream as it is + return CIQ_hash_exists_in_ref(repo, pr_branch, upstream_hash_) + + +def CIQ_commit_exists_in_current_branch(repo, upstream_hash_): + """ + Return True if upstream_hash_ has been backported and it exists in the current branch + """ + + current_branch = CIQ_get_current_branch(repo) + full_upstream_hash = CIQ_get_full_hash(repo, upstream_hash_) + + return CIQ_commit_exists_in_branch(repo, current_branch, full_upstream_hash) + + +def CIQ_find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): + """ + Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive, + if they have not been commited in the pr_branch. + Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length. + Returns a list of tuples: (full_hash, display_string) + """ + results = [] + + # Prepare hash prefixes from 12 down to 6 + hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] + + # Get all commits with 'Fixes:' in the message + output = CIQ_run_git( + repo, + [ + "log", + upstream_ref, + "--grep", + "Fixes:", + "-i", + "--format=%H %h %s (%an)%x0a%B%x00", + ], + ).strip() + if not output: + return [] + + # Each commit is separated by a NUL character and a newline + commits = output.split("\x00\x0a") + for commit in commits: + if not commit.strip(): + continue + + lines = commit.splitlines() + # The first line is the summary, the rest is the body + header = lines[0] + full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split()) + fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:]) + for fix in fixes: + for prefix in hash_prefixes: + if fix.lower().startswith(prefix.lower()): + if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash): + results.append((full_hash, display_string)) + break + + return results + + +def CIQ_find_fixes_in_mainline_current_branch(repo, upstream_ref, hash_): + current_branch = CIQ_get_current_branch(repo) + + return CIQ_find_fixes_in_mainline(repo, current_branch, upstream_ref, hash_) + + def repo_init(repo): """Initialize a git repo object. From b63941bfe81330664fae785b5b90a2ea373a35a7 Mon Sep 17 00:00:00 2001 From: Roxana Nicolescu Date: Mon, 10 Nov 2025 18:54:58 +0100 Subject: [PATCH 7/7] ciq-cherry-pick.py: Automatically cherry pick cve-bf commits It now automatically cherry picks the Fixes: dependencies. To accomodate this, CIQ_find_mainline_fixes was moved to ciq_helpers. And an extra argument for upstream-ref was introduced, the default being origin/kernel-mainline, as the dependencies are looked up there. Signed-off-by: Roxana Nicolescu --- check_kernel_commits.py | 3 +- ciq-cherry-pick.py | 90 ++++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/check_kernel_commits.py b/check_kernel_commits.py index da273d0..ee5450c 100644 --- a/check_kernel_commits.py +++ b/check_kernel_commits.py @@ -11,6 +11,7 @@ from ciq_helpers import ( CIQ_commit_exists_in_branch, CIQ_extract_fixes_references_from_commit_body_lines, + CIQ_find_fixes_in_mainline, CIQ_get_commit_body, CIQ_hash_exists_in_ref, CIQ_run_git, @@ -238,7 +239,7 @@ def main(): ) out_lines.append("") # blank line continue - fixes = find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash) + fixes = CIQ_find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash) if fixes: any_findings = True diff --git a/ciq-cherry-pick.py b/ciq-cherry-pick.py index 5358bbc..6cf6fd9 100644 --- a/ciq-cherry-pick.py +++ b/ciq-cherry-pick.py @@ -1,6 +1,7 @@ import argparse import logging import os +import re import subprocess import git @@ -8,7 +9,9 @@ from ciq_helpers import ( CIQ_cherry_pick_commit_standardization, CIQ_commit_exists_in_current_branch, + CIQ_find_fixes_in_mainline_current_branch, CIQ_fixes_references, + CIQ_get_full_hash, CIQ_original_commit_author_to_tag_string, ) @@ -31,42 +34,17 @@ def check_fixes(sha): raise RuntimeError(f"The commit you want to cherry pick references a Fixes {fix}: but this is not here") -if __name__ == "__main__": - print("CIQ custom cherry picker") - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("--sha", help="Target SHA1 to cherry-pick") - parser.add_argument("--ticket", help="Ticket associated to cherry-pick work, comma separated list is supported.") - parser.add_argument( - "--ciq-tag", - help="Tags for commit message <-optional modifier> .\n" - "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n" - " cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n" - " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" - "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n", - ) - args = parser.parse_args() - +def cherry_pick(sha, ciq_tags, jira_ticket): # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression - git_sha_res = subprocess.run(["git", "show", "--pretty=%H", "-s", args.sha], stdout=subprocess.PIPE) - if git_sha_res.returncode != 0: - print(f"[FAILED] git show --pretty=%H -s {args.sha}") - print("Subprocess Call:") - print(git_sha_res) - print("") - else: - args.sha = git_sha_res.stdout.decode("utf-8").strip() + full_sha = CIQ_get_full_hash(".", sha) - tags = [] - if args.ciq_tag is not None: - tags = args.ciq_tag.split(",") + check_fixes(full_sha) - check_fixes(args.sha) - - author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha) - if author is None: + author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=full_sha) + if author is None: # TODO raise Exception maybe exit(1) - git_res = subprocess.run(["git", "cherry-pick", "-nsx", args.sha]) + git_res = subprocess.run(["git", "cherry-pick", "-nsx", full_sha]) # Move it into a separate method if git_res.returncode != 0: print(f"[FAILED] git cherry-pick -nsx {args.sha}") print(" Manually resolve conflict and include `upstream-diff` tag in commit message") @@ -82,7 +60,7 @@ def check_fixes(sha): with open(MERGE_MSG, "r") as file: original_msg = file.readlines() - new_msg = CIQ_cherry_pick_commit_standardization(original_msg, args.sha, jira=args.ticket, tags=tags) + new_msg = CIQ_cherry_pick_commit_standardization(original_msg, full_sha, jira=jira_ticket, tags=ciq_tags) print(f"Cherry Pick New Message for {args.sha}") for line in new_msg: @@ -94,3 +72,51 @@ def check_fixes(sha): if git_res.returncode == 0: subprocess.run(["git", "commit", "-F", MERGE_MSG]) + + +def cherry_pick_fixes(sha, ciq_tags, jira_ticket, upstream_ref): + fixes_in_mainline = CIQ_find_fixes_in_mainline_current_branch(".", upstream_ref, sha) + + # Replace cve with cve-bf + # Leave cve-pre and cve-bf as they are + ciq_tags = [re.sub(r"^cve ", "cve-bf ", s) for s in ciq_tags] + for full_hash, display_str in fixes_in_mainline: + print(f"Extra cherry picking {display_str}") + full_cherry_pick(sha=full_hash, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref) + + +def full_cherry_pick(sha, ciq_tags, jira_ticket, upstream_ref): + # Cherry pick the commit + cherry_pick(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket) + + # Cherry pick the fixed-by dependencies + cherry_pick_fixes(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref) + + +if __name__ == "__main__": + print("CIQ custom cherry picker") + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("--sha", help="Target SHA1 to cherry-pick") + parser.add_argument("--ticket", help="Ticket associated to cherry-pick work, comma separated list is supported.") + parser.add_argument( + "--ciq-tag", + help="Tags for commit message <-optional modifier> .\n" + "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n" + " cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n" + " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" + "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n", + ) + + parser.add_argument( + "--upstream-ref", + default="origin/kernel-mainline", + help="Reference to upstream mainline branch (default: origin/kernel-mainline)", + ) + + args = parser.parse_args() + + tags = [] + if args.ciq_tag is not None: + tags = args.ciq_tag.split(",") + + full_cherry_pick(sha=args.sha, ciq_tags=tags, jira_ticket=args.ticket, upstream_ref=args.upstream_ref)