From c84937b85292c31411eb2329570fdbf1903582e7 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 5 Apr 2022 15:32:30 +0200 Subject: [PATCH] Add Are We Synapse Yet to GHA (#2321) * Add Are We Synapse Yet to GHA * Better output & add comments --- .github/workflows/dendrite.yml | 9 ++++++++- are-we-synapse-yet.py | 9 +++++---- show-expected-fail-tests.sh | 16 ++++++++-------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 99747f79..c80a82d1 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -233,7 +233,14 @@ jobs: - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal - name: Upload Sytest logs uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index 10b1be28..8d551575 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -3,7 +3,7 @@ from __future__ import division import argparse import re -import sys +import os # Usage: $ ./are-we-synapse-yet.py [-v] results.tap # This script scans a results.tap file from Dendrite's CI process and spits out @@ -156,6 +156,7 @@ def parse_test_line(line): # ✓ POST /register downcases capitals in usernames # ... def print_stats(header_name, gid_to_tests, gid_to_name, verbose): + ci = os.getenv("CI") # When running from GHA, this groups the subsections subsections = [] # Registration: 100% (13/13 tests) subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] total_passing = 0 @@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): for name, passing in tests.items(): if passing: group_passing += 1 - test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") + test_names_and_marks.append(f"{'✅' if passing else '❌'} {name}") total_tests += group_total total_passing += group_passing @@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) print("-" * (len(header_name)+1)) for line in subsections: - print(" %s" % (line,)) + print("%s%s" % ("::group::" if ci and verbose else "", line,)) if verbose: for test_name_and_pass_mark in subsection_test_names[line]: print(" %s" % (test_name_and_pass_mark,)) - print("") + print("%s" % ("::endgroup::" if ci else "")) print("") def main(results_tap_path, verbose): diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 320d4ebd..3ed937a0 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -89,17 +89,17 @@ if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then fi if [ -n "${tests_to_add}" ]; then - echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" - echo "\`\`\`" - echo -e "${tests_to_add}" - echo "\`\`\`" + echo "::error::The following tests passed but are not present in \`$2\`. Please append them to the file:" + echo "::group::Passing tests" + echo -e "${tests_to_add}" + echo "::endgroup::" fi if [ -n "${already_in_whitelist}" ]; then - echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" - echo "\`\`\`" - echo -e "${already_in_whitelist}" - echo "\`\`\`" + echo "::warning::Tests in the whitelist still marked as **expected fail**:" + echo "::group::Still marked as expected fail" + echo -e "${already_in_whitelist}" + echo "::endgroup::" fi exit ${fail_build}