Add a dashboard of build/test status to the nightly build page. (#1340)

* Update build script.

* Add caching.

* Clean up output in text mode.

* Run most basic version of script.

* Untabify

* Fix python script

* Install requests module

* Fix command

* Full script.

* Tabs

* Fix build report.

* Tabs.

* Preserve dashboard comment in test workflow.

* Add script for updating issue comment.

* Update workflow script to handle dashboard.

* Finish script to update issue comment.

* Add issue updating to the workflow.

* Add comments.

* Schedule build matrix earlier.

* Don't include header if --nooutput_header is specified.

* Change heading to Username

* Remove extraneous newline causing bad formatting.

Author: jonsimantov

.github/workflows/build-report.yml

@@ -1,9 +1,9 @@
-name: Generate Weekly Test Report
+name: Generate Test Report Table
 
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 16 * * *"  # 4pm UTC = 8am PST / 9am PDT, 7 hours after testapps run
+    - cron: "0 15 * * *"  # 3pm UTC = 7am PST / 8am PDT, 6 hours after testapps run
 
 env:
   GITHUB_TOKEN: ${{ github.token }}
@@ -14,6 +14,56 @@ jobs:
   generate-report:
     runs-on: ubuntu-20.04
     steps:
+    - name: Setup python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Install Desktop SDK prerequisites
+      uses: nick-invision/retry@v2
+      with:
+        timeout_minutes: 15
+        max_attempts: 3
+        command: |
+          python3 scripts/gha/install_prereqs_desktop.py --gha_build
+          python3 -m pip install requests python-dateutil progress attrs
     - name: Fetch GitHub jobs
       run: |
-        true
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache
+    - name: Generate report files
+      run: |
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --output_markdown --read_cache build_status.cache > report_extended.md
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --nooutput_header --read_cache build_status_short.cache > report.txt
+    - name: Generate comment string
+      run: |
+        echo -n > comment.md
+        cat >> comment.md <<EOF
+
+        ***
+
+        EOF
+        cat report_short.md >> comment.md
+        cat >> comment.md <<EOF
+
+        <details><summary>View extended history</summary>
+
+        EOF
+        cat report_extended.md >> comment.md
+        cat >> comment.md <<EOF
+        </details>
+        <details><summary>📄</summary><pre>
+        EOF
+        cat report.txt >> comment.md
+        cat >> comment.md <<EOF
+        </pre></details>
+        EOF
+    - name: Show comment string
+      run: |
+        cat comment.md
+    - name: Update GitHub issue
+      run: |
+        python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title '[C++] Nightly Integration Testing Report' --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md

scripts/gha/it_workflow.py

@@ -14,7 +14,7 @@
 
 """A utility for integration test workflow.
 
-This script helps to update PR/Issue comments and labels during testing process. 
+This script helps to update PR/Issue comments and labels during testing process.
 
 For PR comment, this script will update (create if not exist) the "Test Result" in comment.
 stage value: [start, progress, end]
@@ -27,7 +27,7 @@
     --run_id ${{github.run_id}} \
     [--new_token ${{steps.generate-token.outputs.token}}]
 
-For Daily Report, this script will update (create if not exist) the "Test Result" in Issue 
+For Daily Report, this script will update (create if not exist) the "Test Result" in Issue
 with title "Nightly Integration Testing Report" and label "nightly-testing".
 stage value: [report]
 USAGE:
@@ -61,8 +61,8 @@
 _LABEL_SUCCEED = "tests: succeeded"
 
 _COMMENT_TITLE_PROGESS = "### ⏳  Integration test in progress...\n"
-_COMMENT_TITLE_PROGESS_FLAKY = "### Integration test with FLAKINESS (but still ⏳  in progress)\n" 
-_COMMENT_TITLE_PROGESS_FAIL = "### ❌  Integration test FAILED (but still ⏳  in progress)\n" 
+_COMMENT_TITLE_PROGESS_FLAKY = "### Integration test with FLAKINESS (but still ⏳  in progress)\n"
+_COMMENT_TITLE_PROGESS_FAIL = "### ❌  Integration test FAILED (but still ⏳  in progress)\n"
 _COMMENT_TITLE_FLAKY = "### Integration test with FLAKINESS (succeeded after retry)\n"
 _COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
@@ -81,6 +81,10 @@
 _COMMENT_IDENTIFIER = "integration-test-status-comment"
 _COMMENT_HIDDEN_DIVIDER = f'\r\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\r\n'
 
+_COMMENT_IDENTIFIER_DASHBOARD = "build-dashboard-comment"
+_COMMENT_DASHBOARD_START = f'\r\n<hidden value="{_COMMENT_IDENTIFIER_DASHBOARD}"-start></hidden>\r\n'
+_COMMENT_DASHBOARD_END = f'\r\n<hidden value="{_COMMENT_IDENTIFIER_DASHBOARD}"-end></hidden>\r\n'
+
 _LOG_ARTIFACT_NAME = "log-artifact"
 _LOG_OUTPUT_DIR = "test_results"
 
@@ -105,7 +109,7 @@
     "Different stage while running the workflow. Valid values in _BUILD_STAGES.")
 
 flags.DEFINE_string(
-    "token", None, 
+    "token", None,
     "github.token: A token to authenticate on your repository.")
 
 flags.DEFINE_string(
@@ -127,7 +131,7 @@
     "new_token", None,
     "Only used with --stage end"
     "Use a different token to remove the \"in-progress\" label,"
-    "to allow the removal to trigger the \"Check Labels\" workflow.")   
+    "to allow the removal to trigger the \"Check Labels\" workflow.")
 
 flags.DEFINE_string(
     "build_against", None,
@@ -151,7 +155,7 @@ def test_start(token, issue_number, actor, commit, run_id):
 
 
 def test_progress(token, issue_number, actor, commit, run_id):
-  """In PR, when some test failed, update failure info and 
+  """In PR, when some test failed, update failure info and
   add label \"tests: failed\""""
   success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
   if success_or_only_flakiness and not log_summary:
@@ -174,7 +178,7 @@ def test_progress(token, issue_number, actor, commit, run_id):
 
 
 def test_end(token, issue_number, actor, commit, run_id, new_token):
-  """In PR, when some test end, update Test Result Report and 
+  """In PR, when some test end, update Test Result Report and
   update label: add \"tests: failed\" if test failed, add label
   \"tests: succeeded\" if test succeed"""
   success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
@@ -205,11 +209,14 @@ def test_end(token, issue_number, actor, commit, run_id, new_token):
 
 
 def test_report(token, actor, commit, run_id, build_against, build_apis):
-  """Update (create if not exist) a Daily/Nightly Report in Issue. 
+  """Update (create if not exist) a Daily/Nightly Report in Issue.
   The Issue with title _REPORT_TITLE and label _REPORT_LABEL:
   https://github.com/firebase/firebase-cpp-sdk/issues?q=is%3Aissue+label%3Anightly-testing
   The report is with the format below:
     PREFIX
+    HIDDEN DASHBOARD START - optional
+    BUILD DASHBOARD - optional
+    HIDDEN DASHBOARD END - optional
     HIDDEN DIVIDER
     REPORT (TEST AGAINST REPO)
     HIDDEN DIVIDER
@@ -222,16 +229,24 @@ def test_report(token, actor, commit, run_id, build_against, build_apis):
     report_title = _REPORT_TITLE
     firestore_issue_number = _get_issue_number(token, _REPORT_TITLE_FIRESTORE, _REPORT_LABEL)
     firestore_issue_url = "https://github.com/firebase/firebase-cpp-sdk/issues/%s" % firestore_issue_number
-    prefix = "Note: This report excludes firestore. Please also check **[the report for firestore](%s)**\n***\n" % firestore_issue_url
+    prefix = "Note: This report excludes Firestore. Please also check **[the report for Firestore](%s).**\n***\n" % firestore_issue_url
 
   issue_number = _get_issue_number(token, report_title, _REPORT_LABEL)
   previous_comment = github.get_issue_body(token, issue_number)
-  [_, previous_comment_repo, previous_comment_sdk, previous_comment_tip] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
+  [previous_prefix, previous_comment_repo, previous_comment_sdk,
+   previous_comment_tip] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
+   # If there is a build dashboard, preserve it.
+   if (_COMMENT_DASHBOARD_START in previous_prefix and
+       _COMMENT_DASHBOARD_END in previous_prefix):
+     [_, previous_dashboard_plus_the_rest] = previous_prefix.split(_COMMENT_DASHBOARD_START)
+     [previous_dashboard, _] = previous_dashboard_plus_the_rest.split(_COMMENT_DASHBOARD_END)
+     prefix = prefix + _COMMENT_DASHBOARD_START + previous_dashboard + _COMMENT_DASHBOARD_END
+
   success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
   if success_or_only_flakiness and not log_summary:
     # succeeded (without flakiness)
     if build_against==_BUILD_AGAINST_REPO:
-      title = _COMMENT_TITLE_SUCCEED_REPO  
+      title = _COMMENT_TITLE_SUCCEED_REPO
     elif build_against==_BUILD_AGAINST_SDK:
       title = _COMMENT_TITLE_SUCCEED_SDK
     else:
@@ -241,21 +256,21 @@ def test_report(token, actor, commit, run_id, build_against, build_apis):
     if success_or_only_flakiness:
       # all failures/errors are due to flakiness (succeeded after retry)
       if build_against==_BUILD_AGAINST_REPO:
-        title = _COMMENT_TITLE_FLAKY_REPO  
+        title = _COMMENT_TITLE_FLAKY_REPO
       elif build_against==_BUILD_AGAINST_SDK:
         title = _COMMENT_TITLE_FLAKY_SDK
       else:
         title = _COMMENT_TITLE_FLAKY_TIP
     else:
       # failures/errors still exist after retry
       if build_against==_BUILD_AGAINST_REPO:
-        title = _COMMENT_TITLE_FAIL_REPO  
+        title = _COMMENT_TITLE_FAIL_REPO
       elif build_against==_BUILD_AGAINST_SDK:
         title = _COMMENT_TITLE_FAIL_SDK
       else:
         title = _COMMENT_TITLE_FAIL_TIP
     comment = title + _get_description(actor, commit, run_id) + log_summary + _COMMENT_FLAKY_TRACKER
-  
+
   if build_against==_BUILD_AGAINST_REPO:
     comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
   elif build_against==_BUILD_AGAINST_SDK:
@@ -267,7 +282,7 @@ def test_report(token, actor, commit, run_id, build_against, build_apis):
     github.close_issue(token, issue_number)
   else:
     github.open_issue(token, issue_number)
-    
+
   github.update_issue_comment(token, issue_number, comment)
 
 
@@ -276,7 +291,13 @@ def _get_issue_number(token, title, label):
   for issue in issues:
     if issue["title"] == title:
       return issue["number"]
-  empty_comment = _COMMENT_HIDDEN_DIVIDER + " " + _COMMENT_HIDDEN_DIVIDER + " " + _COMMENT_HIDDEN_DIVIDER 
+  empty_comment = (" " +
+                   _COMMENT_DASHBOARD_START + " " +
+                   _COMMENT_DASHBOARD_END + " " +
+                   _COMMENT_HIDDEN_DIVIDER + " " +
+                   _COMMENT_HIDDEN_DIVIDER + " " +
+                   _COMMENT_HIDDEN_DIVIDER + " "
+                   )
   return github.create_issue(token, title, label, empty_comment)["number"]
 
 
@@ -287,7 +308,7 @@ def _update_comment(token, issue_number, comment):
   else:
     github.update_comment(token, comment_id, comment)
 
-  
+
 def _get_comment_id(token, issue_number, comment_identifier):
   comments = github.list_comments(token, issue_number)
   for comment in comments: