Update Go, use go tool covdata for coverage files?

This commit is contained in:
Till Faelligen 2023-03-22 14:12:06 +01:00
parent 5e85a00cb3
commit 6948d16527
No known key found for this signature in database
GPG Key ID: ACCDC9606D472758
8 changed files with 30 additions and 28 deletions

View File

@ -12,6 +12,7 @@ concurrency:
jobs:
# run Sytest in different variations
sytest:
if: ${{ false }}
timeout-minutes: 60
name: "Sytest (${{ matrix.label }})"
runs-on: ubuntu-latest
@ -75,7 +76,7 @@ jobs:
name: "Sytest Coverage"
runs-on: ubuntu-latest
needs: sytest # only run once Sytest is done
if: ${{ always() }}
if: ${{ false }}
steps:
- uses: actions/checkout@v3
- name: Install Go
@ -167,7 +168,7 @@ jobs:
cat <<EOF > /tmp/posttest.sh
#!/bin/bash
mkdir -p /tmp/Complement/logs/\$2/\$1/
docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/
EOF
chmod +x /tmp/posttest.sh
@ -190,7 +191,7 @@ jobs:
with:
name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/tmp/Complement/**/complementcover.log
/tmp/Complement/logs/**
complement-coverage:
timeout-minutes: 5
@ -199,7 +200,6 @@ jobs:
needs: complement # only run once Complement is done
if: ${{ always() }}
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
@ -207,20 +207,19 @@ jobs:
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest
- name: Run gocovmerge
- name: Collect coverage
run: |
find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
go tool cover -func=complement.cov
go tool covdata textfmt -i=$(find . -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov
grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: ./complement.cov
files: ./final.cov
flags: complement
fail_ci_if_error: true
element_web:
if: ${{ false }}
timeout-minutes: 120
runs-on: ubuntu-latest
steps:

View File

@ -1,6 +1,6 @@
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
FROM golang:1.20-bullseye as build
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /build
@ -17,7 +17,7 @@ RUN --mount=target=. \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
cp build/scripts/complement-cmd.sh /complement-cmd.sh
WORKDIR /dendrite

View File

@ -1,19 +1,19 @@
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
FROM golang:1.20-bullseye as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
# No password when connecting over localhost
RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \
RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/13/main/pg_hba.conf && \
# Bump up max conns for moar concurrency
sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf
sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf
# This entry script starts postgres, waits for it to be up then starts dendrite
RUN echo '\
#!/bin/bash -eu \n\
pg_lsclusters \n\
pg_ctlcluster 9.6 main start \n\
pg_ctlcluster 13 main start \n\
\n\
until pg_isready \n\
do \n\
@ -35,7 +35,7 @@ RUN --mount=target=. \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \
CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \
cp build/scripts/complement-cmd.sh /complement-cmd.sh
WORKDIR /dendrite

View File

@ -2,14 +2,15 @@
# This script is intended to be used inside a docker container for Complement
export GOCOVERDIR=/tmp/covdatafiles
mkdir -p "${GOCOVERDIR}"
if [[ "${COVER}" -eq 1 ]]; then
echo "Running with coverage"
exec /dendrite/dendrite-cover \
--really-enable-open-registration \
--tls-cert server.crt \
--tls-key server.key \
--config dendrite.yaml \
--test.coverprofile=complementcover.log
--config dendrite.yaml
else
echo "Not running with coverage"
exec /dendrite/dendrite \

View File

@ -18,10 +18,11 @@
package fulltext
import (
"context"
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/matrix-org/dendrite/setup/process"
// side effect imports to allow all possible languages
_ "github.com/blevesearch/bleve/v2/analysis/lang/ar"
_ "github.com/blevesearch/bleve/v2/analysis/lang/cjk"
@ -84,17 +85,18 @@ func (i *IndexElement) SetContentType(v string) {
}
// New opens a new/existing fulltext index
func New(ctx context.Context, cfg config.Fulltext) (fts *Search, err error) {
func New(processCtx *process.ProcessContext, cfg config.Fulltext) (fts *Search, err error) {
fts = &Search{}
fts.FulltextIndex, err = openIndex(cfg)
if err != nil {
return nil, err
}
go func() {
// Wait for the context (should be from process.ProcessContext) to be
// done, indicating that Dendrite is shutting down.
<-ctx.Done()
processCtx.ComponentStarted()
// Wait for the processContext to be done, indicating that Dendrite is shutting down.
<-processCtx.WaitForShutdown()
_ = fts.Close()
processCtx.ComponentFinished()
}()
return fts, nil
}

View File

@ -38,7 +38,7 @@ func mustOpenIndex(t *testing.T, tempDir string) (*fulltext.Search, *process.Pro
cfg.InMemory = false
}
ctx := process.NewProcessContext()
fts, err := fulltext.New(ctx.Context(), cfg)
fts, err := fulltext.New(ctx, cfg)
if err != nil {
t.Fatal("failed to open fulltext index:", err)
}

View File

@ -69,11 +69,10 @@ func AddPublicRoutes(
var fts *fulltext.Search
if dendriteCfg.SyncAPI.Fulltext.Enabled {
fts, err = fulltext.New(processContext.Context(), dendriteCfg.SyncAPI.Fulltext)
fts, err = fulltext.New(processContext, dendriteCfg.SyncAPI.Fulltext)
if err != nil {
logrus.WithError(err).Panicf("failed to create full text")
}
processContext.ComponentStarted()
}
federationPresenceProducer := &producers.FederationAPIPresenceProducer{

View File

@ -51,6 +51,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process.
// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
// the file system event with InMemory=true :(
cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
cfg.SyncAPI.Fulltext.InMemory = true
connStr, closeDb := test.PrepareDBConnectionString(t, dbType)
cfg.Global.DatabaseOptions = config.DatabaseOptions{
@ -70,7 +71,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process.
SingleDatabase: false,
})
cfg.Global.ServerName = "test"
cfg.SyncAPI.Fulltext.InMemory = true
// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
// the file system event with InMemory=true :(
cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)