From 1bcb673e3c0d7bfe8551fd9ae2b4cce84a65e348 Mon Sep 17 00:00:00 2001 From: mujx Date: Tue, 2 Jan 2018 20:26:37 +0200 Subject: [PATCH] Set up a development environment with docker (#387) Signed-off-by: Konstantinos Sideris --- .gitignore | 5 + docker/Dockerfile | 10 ++ docker/README.md | 100 ++++++++++++++ docker/build.sh | 18 +++ docker/dendrite-docker.yml | 126 +++++++++++++++++ docker/docker-compose.yml | 173 ++++++++++++++++++++++++ docker/postgres/create_db.sh | 5 + docker/services/client-api-proxy.sh | 9 ++ docker/services/client-api.sh | 5 + docker/services/federation-api-proxy.sh | 7 + docker/services/federation-api.sh | 5 + docker/services/federation-sender.sh | 5 + docker/services/media-api.sh | 5 + docker/services/monolith.sh | 5 + docker/services/public-rooms-api.sh | 5 + docker/services/room-server.sh | 5 + docker/services/sync-api.sh | 5 + 17 files changed, 493 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/build.sh create mode 100644 docker/dendrite-docker.yml create mode 100644 docker/docker-compose.yml create mode 100644 docker/postgres/create_db.sh create mode 100644 docker/services/client-api-proxy.sh create mode 100644 docker/services/client-api.sh create mode 100644 docker/services/federation-api-proxy.sh create mode 100644 docker/services/federation-api.sh create mode 100644 docker/services/federation-sender.sh create mode 100644 docker/services/media-api.sh create mode 100644 docker/services/monolith.sh create mode 100644 docker/services/public-rooms-api.sh create mode 100644 docker/services/room-server.sh create mode 100644 docker/services/sync-api.sh diff --git a/.gitignore b/.gitignore index 29d28271..b60317da 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,8 @@ _testmain.go *.exe *.test *.prof + +# Generated keys +*.pem +*.key +*.crt diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..6e3955d4 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:alpine3.6 + +RUN mkdir /build + +WORKDIR /build + +RUN apk --update --no-cache add openssl bash git && \ + go get github.com/constabulary/gb/... + +CMD ["bash", "docker/build.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..7d18ce60 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,100 @@ +Development with Docker +--- + +With `docker` and `docker-compose` you can easily spin up a development environment +and start working on dendrite. + +### Requirements + +- docker +- docker-compose (version 3+) + +### Configuration + +Copy the `dendrite-docker.yaml` file to the root of the project and rename it to +`dendrite.yaml`. It already contains the defaults used in `docker-compose` for +networking so you will only have to change things like the `server_name` or to +toggle `naffka`. + +You can run the following `docker-compose` commands either from the top directory +specifying the `docker-compose` file +``` +docker-compose -f docker/docker-compose.yml +``` +or from within the `docker` directory + +``` +docker-compose +``` + +### Starting a monolith server + +For the monolith server you would need a postgres instance + +``` +docker-compose up postgres +``` + +and the dendrite component from `bin/dendrite-monolith-server` + +``` +docker-compose up monolith +``` + +The monolith will be listening on `http://localhost:8008`. + +You would also have to make the following adjustments to `dendrite.yaml`. + - Set `use_naffka: true` + - Uncomment the `database/naffka` postgres url. + +### Starting a multiprocess server + +The multiprocess server requires kafka, zookeeper and postgres + +``` +docker-compose up kafka zookeeper postgres +``` + +and the following dendrite components + +``` +docker-compose up client_api media_api sync_api room_server public_rooms_api +docker-compose up client_api_proxy +``` + +The `client-api-proxy` will be listening on `http://localhost:8008`. + +You would also have to make the following adjustments to `dendrite.yaml`. + - Set `use_naffka: false` + - Comment out the `database/naffka` postgres url. + +### Starting federation + +``` +docker-compose up federation_api federation_sender +docker-compose up federation_api_proxy +``` + +You can point other Matrix servers to `http://localhost:8448`. + +### Creating a new component + +You can create a new dendrite component by adding an entry to the `docker-compose.yml` +file and creating a startup script for the component in `docker/services`. +For more information refer to the official docker-compose [documentation](https://docs.docker.com/compose/). + +```yaml + new_component: + container_name: dendrite_room_server + hostname: new_component + # Start up script. + entrypoint: ["bash", "./docker/services/new-component.sh"] + # Use the common Dockerfile for all the dendrite components. + build: ./ + volumes: + - ..:/build + depends_on: + - another_component + networks: + - internal +``` diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 00000000..a478c867 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +gb build + +# Generate the keys if they don't already exist. +if [ ! -f server.key ] || [ ! -f server.crt ] || [ ! -f matrix_key.pem ]; then + echo "Generating keys ..." + + rm -f server.key server.crt matrix_key.pem + + test -f server.key || openssl req -x509 -newkey rsa:4096 \ + -keyout server.key \ + -out server.crt \ + -days 3650 -nodes \ + -subj /CN=localhost + + test -f matrix_key.pem || /build/bin/generate-keys -private-key matrix_key.pem +fi diff --git a/docker/dendrite-docker.yml b/docker/dendrite-docker.yml new file mode 100644 index 00000000..024eee72 --- /dev/null +++ b/docker/dendrite-docker.yml @@ -0,0 +1,126 @@ +# The config file format version +# This is used by dendrite to tell if it understands the config format. +# This will change if the structure of the config file changes or if the meaning +# of an existing config key changes. +version: 0 + +# The matrix specific config +matrix: + # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + server_name: "example.com" + # The path to the PEM formatted matrix private key. + private_key: "matrix_key.pem" + # The x509 certificates used by the federation listeners for this server + federation_certificates: ["server.crt"] + # The list of identity servers trusted to verify third party identifiers by this server. + # Defaults to no trusted servers. + trusted_third_party_id_servers: + - vector.im + - matrix.org + - riot.im + +# The media repository config +media: + # The base path to where the media files will be stored. May be relative or absolute. + base_path: /var/dendrite/media + + # The maximum file size in bytes that is allowed to be stored on this server. + # Note: if max_file_size_bytes is set to 0, the size is unlimited. + # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + max_file_size_bytes: 10485760 + + # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + # NOTE: This is a possible denial-of-service attack vector - use at your own risk + dynamic_thumbnails: false + + # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + # method is one of crop or scale. If omitted, it will default to scale. + # crop scales to fill the requested dimensions and crops the excess. + # scale scales to fit the requested dimensions and one dimension may be smaller than requested. + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale + +# The config for the TURN server +turn: + # Whether or not guests can request TURN credentials + turn_allow_guests: true + # How long the authorization should last + turn_user_lifetime: "1h" + # The list of TURN URIs to pass to clients + turn_uris: [] + + # Authorization via Shared Secret + # The shared secret from coturn + turn_shared_secret: "" + + # Authorization via Static Username & Password + # Hardcoded Username and Password + turn_username: "" + turn_password: "" + +# The config for communicating with kafka +kafka: + # Where the kafka servers are running. + addresses: ["kafka:9092"] + # Whether to use naffka instead of kafka. + # Naffka can only be used when running dendrite as a single monolithic server. + # Kafka can be used both with a monolithic server and when running the + # components as separate servers. + # If enabled database.naffka must also be specified. + use_naffka: true + # The names of the kafka topics to use. + topics: + output_room_event: roomserverOutput + output_client_data: clientapiOutput + user_updates: userUpdates + +# The postgres connection configs for connecting to the databases e.g a postgres:// URI +database: + account: "postgres://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable" + device: "postgres://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable" + media_api: "postgres://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable" + sync_api: "postgres://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable" + room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable" + server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable" + federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable" + public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable" + # If using naffka you need to specify a naffka database + naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable" + +# The TCP host:port pairs to bind the internal HTTP APIs to. +# These shouldn't be exposed to the public internet. +# These aren't needed when running dendrite as a monolithic server. +listen: + room_server: "room_server:7770" + client_api: "client_api:7771" + federation_api: "federation_api:7772" + sync_api: "sync_api:7773" + media_api: "media_api:7774" + public_rooms_api: "public_rooms_api:7775" + federation_sender: "federation_sender:7776" + +# The configuration for tracing the dendrite components. +tracing: + # Config for the jaeger opentracing reporter. + # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration + # for documtation. + jaeger: + disabled: true + +# A list of application service config files to use +application_services: + config_files: [] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..9d4312fb --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,173 @@ +version: "3.4" +services: + monolith: + container_name: dendrite_monolith + hostname: monolith + entrypoint: ["bash", "./docker/services/monolith.sh"] + build: ./ + volumes: + - ..:/build + networks: + - internal + depends_on: + - postgres + ports: + - "8008:8008" + - "8448:8448" + + client_api_proxy: + container_name: dendrite_client_api_proxy + hostname: client_api_proxy + entrypoint: ["bash", "./docker/services/client-api-proxy.sh"] + build: ./ + volumes: + - ..:/build + networks: + - internal + depends_on: + - postgres + - sync_api + - client_api + - media_api + - public_rooms_api + ports: + - "8008:8008" + + client_api: + container_name: dendrite_client_api + hostname: client_api + entrypoint: ["bash", "./docker/services/client-api.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + - room_server + networks: + - internal + + media_api: + container_name: dendrite_media_api + hostname: media_api + entrypoint: ["bash", "./docker/services/media-api.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + networks: + - internal + + public_rooms_api: + container_name: dendrite_public_rooms_api + hostname: public_rooms_api + entrypoint: ["bash", "./docker/services/public-rooms-api.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + networks: + - internal + + sync_api: + container_name: dendrite_sync_api + hostname: sync_api + entrypoint: ["bash", "./docker/services/sync-api.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + networks: + - internal + + room_server: + container_name: dendrite_room_server + hostname: room_server + entrypoint: ["bash", "./docker/services/room-server.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + networks: + - internal + + federation_api_proxy: + container_name: dendrite_federation_api_proxy + hostname: federation_api_proxy + entrypoint: ["bash", "./docker/services/federation-api-proxy.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + - federation_api + - federation_sender + - media_api + networks: + - internal + ports: + - "8448:8448" + + federation_api: + container_name: dendrite_federation_api + hostname: federation_api + entrypoint: ["bash", "./docker/services/federation-api.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + networks: + - internal + + federation_sender: + container_name: dendrite_federation_sender + hostname: federation_sender + entrypoint: ["bash", "./docker/services/federation-sender.sh"] + build: ./ + volumes: + - ..:/build + depends_on: + - postgres + networks: + - internal + + postgres: + container_name: dendrite_postgres + hostname: postgres + image: postgres:9.5 + restart: always + volumes: + - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh + environment: + POSTGRES_PASSWORD: itsasecret + POSTGRES_USER: dendrite + networks: + - internal + + zookeeper: + container_name: dendrite_zk + hostname: zookeeper + image: zookeeper + networks: + - internal + + kafka: + container_name: dendrite_kafka + hostname: kafka + image: wurstmeister/kafka + environment: + KAFKA_ADVERTISED_HOST_NAME: "kafka" + KAFKA_DELETE_TOPIC_ENABLE: "true" + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + depends_on: + - zookeeper + networks: + - internal + +networks: + internal: + attachable: true diff --git a/docker/postgres/create_db.sh b/docker/postgres/create_db.sh new file mode 100644 index 00000000..56f6540e --- /dev/null +++ b/docker/postgres/create_db.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka; do + createdb -O dendrite dendrite_$db +done diff --git a/docker/services/client-api-proxy.sh b/docker/services/client-api-proxy.sh new file mode 100644 index 00000000..931f7abb --- /dev/null +++ b/docker/services/client-api-proxy.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/client-api-proxy --bind-address ":8008" \ + --client-api-server-url "http://client_api:7771" \ + --sync-api-server-url "http://sync_api:7773" \ + --media-api-server-url "http://media_api:7774" \ + --public-rooms-api-server-url "http://public_rooms_api:7775" \ diff --git a/docker/services/client-api.sh b/docker/services/client-api.sh new file mode 100644 index 00000000..8dc82242 --- /dev/null +++ b/docker/services/client-api.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-client-api-server --config=dendrite.yaml diff --git a/docker/services/federation-api-proxy.sh b/docker/services/federation-api-proxy.sh new file mode 100644 index 00000000..6ea75c95 --- /dev/null +++ b/docker/services/federation-api-proxy.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/federation-api-proxy --bind-address ":8448" \ + --federation-api-url "http://federation_api_server:7772" \ + --media-api-server-url "http://media_api:7774" \ diff --git a/docker/services/federation-api.sh b/docker/services/federation-api.sh new file mode 100644 index 00000000..807a7cf8 --- /dev/null +++ b/docker/services/federation-api.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-federation-api-server --config dendrite.yaml diff --git a/docker/services/federation-sender.sh b/docker/services/federation-sender.sh new file mode 100644 index 00000000..ea116ef3 --- /dev/null +++ b/docker/services/federation-sender.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-federation-sender-server --config dendrite.yaml diff --git a/docker/services/media-api.sh b/docker/services/media-api.sh new file mode 100644 index 00000000..876b3aa8 --- /dev/null +++ b/docker/services/media-api.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-media-api-server --config dendrite.yaml diff --git a/docker/services/monolith.sh b/docker/services/monolith.sh new file mode 100644 index 00000000..a038728b --- /dev/null +++ b/docker/services/monolith.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key diff --git a/docker/services/public-rooms-api.sh b/docker/services/public-rooms-api.sh new file mode 100644 index 00000000..652afcfe --- /dev/null +++ b/docker/services/public-rooms-api.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-public-rooms-api-server --config dendrite.yaml diff --git a/docker/services/room-server.sh b/docker/services/room-server.sh new file mode 100644 index 00000000..473b5f5d --- /dev/null +++ b/docker/services/room-server.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-room-server --config=dendrite.yaml diff --git a/docker/services/sync-api.sh b/docker/services/sync-api.sh new file mode 100644 index 00000000..ac6433fa --- /dev/null +++ b/docker/services/sync-api.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-sync-api-server --config=dendrite.yaml