Set up a development environment with docker (#387)

Signed-off-by: Konstantinos Sideris <sideris.konstantin@gmail.com>

.gitignore

@@ -34,3 +34,8 @@ _testmain.go
 *.exe
 *.test
 *.prof
+
+# Generated keys
+*.pem
+*.key
+*.crt

docker/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:alpine3.6
+
+RUN mkdir /build
+
+WORKDIR /build
+
+RUN apk --update --no-cache add openssl bash git && \
+    go get github.com/constabulary/gb/...
+
+CMD ["bash", "docker/build.sh"]

docker/README.md

@@ -0,0 +1,100 @@
+Development with Docker
+---
+
+With `docker` and `docker-compose` you can easily spin up a development environment
+and start working on dendrite.
+
+### Requirements
+
+- docker
+- docker-compose (version 3+)
+
+### Configuration
+
+Copy the `dendrite-docker.yaml` file to the root of the project and rename it to
+`dendrite.yaml`. It already contains the defaults used in `docker-compose` for 
+networking so you will only have to change things like the `server_name` or to 
+toggle `naffka`.
+
+You can run the following `docker-compose` commands either from the top directory
+specifying the `docker-compose` file
+```
+docker-compose -f docker/docker-compose.yml <cmd>
+```
+or from within the `docker` directory 
+
+```
+docker-compose <cmd>
+```
+
+### Starting a monolith server
+
+For the monolith server you would need a postgres instance 
+
+```
+docker-compose up postgres
+```
+
+and the dendrite component from `bin/dendrite-monolith-server`
+
+```
+docker-compose up monolith
+```
+
+The monolith will be listening on `http://localhost:8008`.
+
+You would also have to make the following adjustments to `dendrite.yaml`.
+ - Set `use_naffka: true` 
+ - Uncomment the `database/naffka` postgres url.
+
+### Starting a multiprocess server
+
+The multiprocess server requires kafka, zookeeper and postgres
+
+```
+docker-compose up kafka zookeeper postgres
+```
+
+and the following dendrite components 
+
+```
+docker-compose up client_api media_api sync_api room_server public_rooms_api
+docker-compose up client_api_proxy
+```
+
+The `client-api-proxy` will be listening on `http://localhost:8008`.
+
+You would also have to make the following adjustments to `dendrite.yaml`.
+ - Set `use_naffka: false` 
+ - Comment out the `database/naffka` postgres url.
+
+### Starting federation
+
+```
+docker-compose up federation_api federation_sender
+docker-compose up federation_api_proxy
+```
+
+You can point other Matrix servers to `http://localhost:8448`.
+
+### Creating a new component
+
+You can create a new dendrite component by adding an entry to the `docker-compose.yml` 
+file and creating a startup script for the component in `docker/services`. 
+For more information refer to the official docker-compose [documentation](https://docs.docker.com/compose/).
+
+```yaml
+  new_component:
+    container_name: dendrite_room_server
+    hostname: new_component
+    # Start up script.
+    entrypoint: ["bash", "./docker/services/new-component.sh"]
+    # Use the common Dockerfile for all the dendrite components.
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - another_component
+    networks:
+      - internal
+```

docker/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+gb build
+
+# Generate the keys if they don't already exist.
+if [ ! -f server.key ] || [ ! -f server.crt ] || [ ! -f matrix_key.pem ]; then
+    echo "Generating keys ..."
+
+    rm -f server.key server.crt matrix_key.pem
+
+    test -f server.key || openssl req -x509 -newkey rsa:4096 \
+                        -keyout server.key \
+                        -out server.crt \
+                        -days 3650 -nodes \
+                        -subj /CN=localhost 
+
+    test -f matrix_key.pem || /build/bin/generate-keys -private-key matrix_key.pem
+fi

docker/dendrite-docker.yml

@@ -0,0 +1,126 @@
+# The config file format version
+# This is used by dendrite to tell if it understands the config format.
+# This will change if the structure of the config file changes or if the meaning
+# of an existing config key changes.
+version: 0
+
+# The matrix specific config
+matrix:
+    # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
+    server_name: "example.com"
+    # The path to the PEM formatted matrix private key.
+    private_key: "matrix_key.pem"
+    # The x509 certificates used by the federation listeners for this server
+    federation_certificates: ["server.crt"]
+    # The list of identity servers trusted to verify third party identifiers by this server.
+    # Defaults to no trusted servers.
+    trusted_third_party_id_servers:
+      - vector.im
+      - matrix.org
+      - riot.im
+
+# The media repository config
+media:
+    # The base path to where the media files will be stored. May be relative or absolute.
+    base_path: /var/dendrite/media
+
+    # The maximum file size in bytes that is allowed to be stored on this server.
+    # Note: if max_file_size_bytes is set to 0, the size is unlimited.
+    # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
+    max_file_size_bytes: 10485760
+
+    # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
+    # NOTE: This is a possible denial-of-service attack vector - use at your own risk
+    dynamic_thumbnails: false
+
+    # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
+    # method is one of crop or scale. If omitted, it will default to scale.
+    # crop scales to fill the requested dimensions and crops the excess.
+    # scale scales to fit the requested dimensions and one dimension may be smaller than requested.
+    thumbnail_sizes:
+      - width: 32
+        height: 32
+        method: crop
+      - width: 96
+        height: 96
+        method: crop
+      - width: 320
+        height: 240
+        method: scale
+      - width: 640
+        height: 480
+        method: scale
+      - width: 800
+        height: 600
+        method: scale
+
+# The config for the TURN server
+turn:
+    # Whether or not guests can request TURN credentials
+    turn_allow_guests: true
+    # How long the authorization should last
+    turn_user_lifetime: "1h"
+    # The list of TURN URIs to pass to clients
+    turn_uris: []
+
+    # Authorization via Shared Secret
+    # The shared secret from coturn
+    turn_shared_secret: "<SECRET STRING GOES HERE>"
+
+    # Authorization via Static Username & Password
+    # Hardcoded Username and Password
+    turn_username: ""
+    turn_password: ""
+
+# The config for communicating with kafka
+kafka:
+    # Where the kafka servers are running.
+    addresses: ["kafka:9092"]
+    # Whether to use naffka instead of kafka.
+    # Naffka can only be used when running dendrite as a single monolithic server.
+    # Kafka can be used both with a monolithic server and when running the
+    # components as separate servers.
+    # If enabled database.naffka must also be specified.
+    use_naffka: true
+    # The names of the kafka topics to use.
+    topics:
+        output_room_event: roomserverOutput
+        output_client_data: clientapiOutput
+        user_updates: userUpdates
+
+# The postgres connection configs for connecting to the databases e.g a postgres:// URI
+database:
+    account: "postgres://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable"
+    device: "postgres://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable"
+    media_api: "postgres://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable"
+    sync_api: "postgres://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable"
+    room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable"
+    server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable"
+    federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable"
+    public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable"
+    # If using naffka you need to specify a naffka database
+    naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable"
+
+# The TCP host:port pairs to bind the internal HTTP APIs to.
+# These shouldn't be exposed to the public internet.
+# These aren't needed when running dendrite as a monolithic server.
+listen:
+    room_server: "room_server:7770"
+    client_api: "client_api:7771"
+    federation_api: "federation_api:7772"
+    sync_api: "sync_api:7773"
+    media_api: "media_api:7774"
+    public_rooms_api: "public_rooms_api:7775"
+    federation_sender: "federation_sender:7776"
+
+# The configuration for tracing the dendrite components.
+tracing:
+    # Config for the jaeger opentracing reporter.
+    # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration
+    # for documtation.
+    jaeger:
+        disabled: true
+
+# A list of application service config files to use
+application_services:
+    config_files: []

docker/docker-compose.yml

@@ -0,0 +1,173 @@
+version: "3.4"
+services:
+  monolith:
+    container_name: dendrite_monolith
+    hostname: monolith
+    entrypoint: ["bash", "./docker/services/monolith.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    networks:
+      - internal
+    depends_on:
+      - postgres
+    ports:
+      - "8008:8008"
+      - "8448:8448"
+
+  client_api_proxy:
+    container_name: dendrite_client_api_proxy
+    hostname: client_api_proxy
+    entrypoint: ["bash", "./docker/services/client-api-proxy.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    networks:
+      - internal
+    depends_on:
+      - postgres
+      - sync_api
+      - client_api
+      - media_api
+      - public_rooms_api
+    ports:
+      - "8008:8008"
+
+  client_api:
+    container_name: dendrite_client_api
+    hostname: client_api
+    entrypoint: ["bash", "./docker/services/client-api.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+      - room_server
+    networks:
+      - internal
+
+  media_api:
+    container_name: dendrite_media_api
+    hostname: media_api
+    entrypoint: ["bash", "./docker/services/media-api.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+    networks:
+      - internal
+
+  public_rooms_api:
+    container_name: dendrite_public_rooms_api
+    hostname: public_rooms_api
+    entrypoint: ["bash", "./docker/services/public-rooms-api.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+    networks:
+      - internal
+
+  sync_api:
+    container_name: dendrite_sync_api
+    hostname: sync_api
+    entrypoint: ["bash", "./docker/services/sync-api.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+    networks:
+      - internal
+
+  room_server:
+    container_name: dendrite_room_server
+    hostname: room_server
+    entrypoint: ["bash", "./docker/services/room-server.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+    networks:
+      - internal
+
+  federation_api_proxy:
+    container_name: dendrite_federation_api_proxy
+    hostname: federation_api_proxy
+    entrypoint: ["bash", "./docker/services/federation-api-proxy.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+      - federation_api
+      - federation_sender
+      - media_api
+    networks:
+      - internal
+    ports:
+      - "8448:8448"
+
+  federation_api:
+    container_name: dendrite_federation_api
+    hostname: federation_api
+    entrypoint: ["bash", "./docker/services/federation-api.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+    networks:
+      - internal
+
+  federation_sender:
+    container_name: dendrite_federation_sender
+    hostname: federation_sender
+    entrypoint: ["bash", "./docker/services/federation-sender.sh"]
+    build: ./
+    volumes:
+      - ..:/build
+    depends_on:
+      - postgres
+    networks:
+      - internal
+
+  postgres:
+    container_name: dendrite_postgres
+    hostname: postgres
+    image: postgres:9.5
+    restart: always
+    volumes:
+      - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+    networks:
+      - internal
+
+  zookeeper:
+    container_name: dendrite_zk
+    hostname: zookeeper
+    image: zookeeper
+    networks:
+      - internal
+
+  kafka:
+    container_name: dendrite_kafka
+    hostname: kafka
+    image: wurstmeister/kafka
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: "kafka"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+    depends_on:
+      - zookeeper
+    networks:
+      - internal
+
+networks:
+  internal:
+    attachable: true

docker/postgres/create_db.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka; do
+    createdb -O dendrite dendrite_$db
+done

docker/services/client-api-proxy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/client-api-proxy --bind-address ":8008" \
+    --client-api-server-url "http://client_api:7771" \
+    --sync-api-server-url "http://sync_api:7773" \
+    --media-api-server-url "http://media_api:7774" \
+    --public-rooms-api-server-url "http://public_rooms_api:7775" \

docker/services/client-api.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-client-api-server --config=dendrite.yaml

docker/services/federation-api-proxy.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/federation-api-proxy --bind-address ":8448" \
+    --federation-api-url "http://federation_api_server:7772" \
+    --media-api-server-url "http://media_api:7774" \

docker/services/federation-api.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-federation-api-server --config dendrite.yaml

docker/services/federation-sender.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-federation-sender-server --config dendrite.yaml

docker/services/media-api.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-media-api-server --config dendrite.yaml

docker/services/monolith.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key

docker/services/public-rooms-api.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-public-rooms-api-server --config dendrite.yaml

docker/services/room-server.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-room-server --config=dendrite.yaml

docker/services/sync-api.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+bash ./docker/build.sh
+
+./bin/dendrite-sync-api-server --config=dendrite.yaml