dendrite/.github/workflows/schedules.yaml
Till baf118b08c
Add Sytest/Complement coverage to scheduled runs (#2962)
This adds Sytest and Complement coverage reporting to the nightly
scheduled CI runs.

Fixes a few API mode related issues as well, since we seemingly never
really ran them with Complement.

Also fixes a bug related to device list changes: When we pass in an
empty `newlyLeftRooms` slice, we got a list of all currently joined
rooms with the corresponding members. When we then got the
`newlyJoinedRooms`, we wouldn't update the `changed` slice, because we
already got the user from the `newlyLeftRooms` query. This is fixed by
simply ignoring empty `newlyLeftRooms`.
2023-02-03 13:42:35 +01:00

286 lines
9.6 KiB
YAML

name: Scheduled
on:
schedule:
- cron: '0 0 * * *' # every day at midnight
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# run Sytest in different variations
sytest:
timeout-minutes: 60
name: "Sytest (${{ matrix.label }})"
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- label: SQLite native
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: postgres
- label: PostgreSQL, full HTTP APIs
postgres: postgres
api: full-http
container:
image: matrixdotorg/sytest-dendrite:latest
volumes:
- ${{ github.workspace }}:/src
- /root/.cache/go-build:/github/home/.cache/go-build
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
RACE_DETECTION: 1
COVER: 1
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
/gopath/pkg/mod
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-sytest-
- name: Run Sytest
run: /bootstrap.sh dendrite
working-directory: /src
- name: Summarise results.tap
if: ${{ always() }}
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
- name: Sytest List Maintenance
if: ${{ always() }}
run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
continue-on-error: true # not fatal
- name: Are We Synapse Yet?
if: ${{ always() }}
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/logs/results.tap
/logs/**/*.log*
sytest-coverage:
timeout-minutes: 5
name: "Sytest Coverage"
runs-on: ubuntu-latest
needs: sytest # only run once Sytest is done
if: ${{ always() }}
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest
- name: Run gocovmerge
run: |
find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
go tool cover -func=sytest.cov
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: ./sytest.cov
flags: sytest
fail_ci_if_error: true
# run Complement
complement:
name: "Complement (${{ matrix.label }})"
timeout-minutes: 60
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- label: SQLite native
cgo: 0
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
cgo: 0
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: Postgres
cgo: 0
- label: PostgreSQL, full HTTP APIs
postgres: Postgres
api: full-http
cgo: 0
steps:
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
- name: "Set Go Version"
run: |
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
echo "~/go/bin" >> $GITHUB_PATH
- name: "Install Complement Dependencies"
# We don't need to install Go because it is included on the Ubuntu 20.04 image:
# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v3
with:
path: dendrite
# Attempt to check out the same branch of Complement as the PR. If it
# doesn't exist, fallback to main.
- name: Checkout complement
shell: bash
run: |
mkdir -p complement
# Attempt to use the version of complement which best matches the current
# build. Depending on whether this is a PR or release, etc. we need to
# use different fallbacks.
#
# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
# for pull requests, otherwise GITHUB_REF).
# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
# (GITHUB_BASE_REF for pull requests).
# 3. Use the default complement branch ("master").
for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
# Skip empty branch names and merge commits.
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
continue
fi
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
- name: Create post test script
run: |
cat <<EOF > /tmp/posttest.sh
#!/bin/bash
mkdir -p /tmp/Complement/logs/\$2/\$1/
docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
EOF
chmod +x /tmp/posttest.sh
# Run Complement
- run: |
set -o pipefail &&
go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
shell: bash
name: Run Complement Tests
env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
COMPLEMENT_DENDRITE_COVER: 1
COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
working-directory: complement
- name: Upload Complement logs
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/tmp/Complement/**/complementcover.log
complement-coverage:
timeout-minutes: 5
name: "Complement Coverage"
runs-on: ubuntu-latest
needs: complement # only run once Complement is done
if: ${{ always() }}
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest
- name: Run gocovmerge
run: |
find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
go tool cover -func=complement.cov
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: ./complement.cov
flags: complement
fail_ci_if_error: true
element_web:
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
- uses: tecolicom/actions-use-apt-tools@v1
with:
# Our test suite includes some screenshot tests with unusual diacritics, which are
# supposed to be covered by STIXGeneral.
tools: fonts-stix
- uses: actions/checkout@v2
with:
repository: matrix-org/matrix-react-sdk
- uses: actions/setup-node@v3
with:
cache: 'yarn'
- name: Fetch layered build
run: scripts/ci/layered.sh
- name: Copy config
run: cp element.io/develop/config.json config.json
working-directory: ./element-web
- name: Build
env:
CI_PACKAGE: true
run: yarn build
working-directory: ./element-web
- name: Edit Test Config
run: |
sed -i '/HOMESERVER/c\ HOMESERVER: "dendrite",' cypress.config.ts
- name: "Run cypress tests"
uses: cypress-io/github-action@v4.1.1
with:
browser: chrome
start: npx serve -p 8080 ./element-web/webapp
wait-on: 'http://localhost:8080'
env:
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
TMPDIR: ${{ runner.temp }}