diff --git a/.github/workflows/verify-changelog-and-set-milestone.yml b/.github/workflows/verify-changelog-and-set-milestone.yml
index 3427c5a9cefc..1d544be3d12c 100644
--- a/.github/workflows/verify-changelog-and-set-milestone.yml
+++ b/.github/workflows/verify-changelog-and-set-milestone.yml
@@ -56,9 +56,8 @@ jobs:
 
           echo "Diff:"
           if git diff --exit-code --name-only "$BASE_COMMIT" HEAD -- "$CHANGE_LOG_FILE"; then
-            echo "Change log file:$CHANGE_LOG_FILE does not contains an entry corresponding to changes introduced in PR. Please add a changelog entry."
-            gh pr comment "$PR_NUMBER" --body "This PR does not have an entry in lucene/CHANGES.txt. Consider adding one. If the PR doesn't need a changelog entry, then add the skip-changelog label to it and you will stop receiving this reminder on future updates to the PR."
-            exit 0
+            echo "::error ::Change log file:$CHANGE_LOG_FILE does not contains an entry corresponding to changes introduced in PR. Please add a changelog entry."
+            exit 1
           else
             echo "$CHANGE_LOG_FILE contains change log entry. Proceeding with next steps."
           fi