ci: test environment docker image for CI pipelines

This commit includes a Dockerfile for building a test environment for
future CI pipelines. This commit also refactors the GitLab CI pipeline
files by splitting the jobs into different files:

- .gitlab-ci.yml: Global CI pipeline file.
- .gitlab/ci/test-env.gitlab-ci.yml: Jobs to test and publish the
docker image for the test environment.
- .gitlab/ci/playbook.gitlab-ci.yml: Jobs to test and publish the
docker image for the pleroma playbook
- .gitlab/ci/templates/docker.gitlab-ci.yml: Template jobs for
testing and publishing docker images.

Part of dananglin/pleroma-ansible-playbook#17

.dockerignore

@@ -0,0 +1,5 @@
+*
+!roles
+!library
+!Makefile
+!playbook.yml

.gitignore

@@ -1,6 +1,5 @@
 inventories/*
 !inventories/.gitkeep
-site.yml
 vapid-private-key.pem
 
 library/__pycache__/

.gitlab-ci.yml

@@ -1,13 +1,15 @@
 ---
-image: python:3.7.6-slim-buster
+workflow:
+  rules:
+  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+  - if: '$CI_COMMIT_TAG'
+  - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
 
 stages:
 - test
+- publish
 
-test:
-  stage: test
-  before_script:
-  - apt-get update && apt-get install make
-  - pip install ansible==2.9.6
-  script:
-  - make test_modules_unit
+include:
+- local: '/.gitlab/ci/templates/docker.gitlab-ci.yml'
+- local: '/.gitlab/ci/test-env.gitlab-ci.yml'
+- local: '/.gitlab/ci/playbook.gitlab-ci.yml'

.gitlab/ci/playbook.gitlab-ci.yml

@@ -0,0 +1,60 @@
+---
+.use-python:
+  image: python:3.7.6-slim-buster
+
+.playbook-docker-vars:
+  variables:
+    DOCKERFILE: "Dockerfile"
+    IMAGE_NAME: ${CI_REGISTRY}/${CI_PROJECT_PATH}/playbook
+    IMAGE_TAG: ${CI_COMMIT_REF_NAME}
+
+.playbook-docker-test-rules:
+  rules:
+  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    changes:
+    - "Dockerfile"
+    when: always
+
+test:playbook:dockerfile-lint:
+  extends:
+  - .dockerfile-lint
+  - .playbook-docker-vars
+  - .playbook-docker-test-rules
+
+test:playbook:docker-build:
+  extends:
+  - .dockerbuild-test
+  - .playbook-docker-vars
+  - .playbook-docker-test-rules
+  script:
+  - export IMAGE_DATE=$(date -Iseconds)
+  - export BUILD_ARGS="--build-arg BUILD_TAG=${IMAGE_TAG} --build-arg BUILD_TIME=${IMAGE_DATE}"
+  - make image
+
+test:playbook:custom-modules:
+  stage: test
+  extends: .use-python
+  before_script:
+  - apt-get update && apt-get install make
+  - pip install ansible==2.9.7
+  script:
+  - make test_modules_unit
+  only:
+    refs:
+    - merge_requests
+    changes:
+    - library/*
+  except:
+    refs:
+    - master
+
+publish:playbook:docker-image:
+  extends:
+  - .docker-publish
+  - .playbook-docker-vars
+  script:
+  - export IMAGE_DATE=$(date -Iseconds)
+  - export BUILD_ARGS="--build-arg BUILD_TAG=${IMAGE_TAG} --build-arg BUILD_TIME=${IMAGE_DATE}"
+  - make publish
+  only:
+  - /^v[0-9]+(.[0-9]+){2}$/

.gitlab/ci/templates/docker.gitlab-ci.yml

@@ -0,0 +1,29 @@
+---
+.install-make: &install-make
+  - apk add --no-cache make
+
+.docker-build-setup:
+  image: docker:19.03.8
+  services:
+  - docker:19.03.8-dind
+
+.dockerfile-lint:
+  stage: test
+  image: hadolint/hadolint:v1.18.0-alpine
+  script:
+  - hadolint ${DOCKERFILE}
+
+.dockerbuild-test:
+  stage: test
+  extends: .docker-build-setup
+  before_script:
+  - *install-make
+
+.docker-publish:
+  stage: publish
+  extends: .docker-build-setup
+  before_script:
+  - *install-make
+  - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
+  after_script:
+  - docker logout ${CI_REGISTRY}

.gitlab/ci/test-env.gitlab-ci.yml

@@ -0,0 +1,39 @@
+---
+.test-env-docker-vars:
+  variables:
+    IMAGE_NAME: ${CI_REGISTRY}/${CI_PROJECT_PATH}/test-environment
+    IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}
+    DOCKERFILE: "test/pleroma_test_env/Dockerfile"
+    DOCKER_CONTEXT: "test/pleroma_test_env"
+    BUILD_ARGS: "--build-arg TEST_ENV_PASSWORD=${TEST_ENV_PASSWORD}"
+
+.test-env-test-rules:
+  rules:
+  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    changes:
+    - "test/pleroma_test_env/Dockerfile"
+    when: always
+
+test:test-env:dockerfile-lint:
+  extends:
+  - .dockerfile-lint
+  - .test-env-docker-vars
+  - .test-env-test-rules
+
+test:test-env:docker-build:
+  extends:
+  - .dockerbuild-test
+  - .test-env-docker-vars
+  - .test-env-test-rules
+  script:
+  - make image
+
+publish:test-env:docker-image:
+  extends:
+  - .docker-publish
+  - .test-env-docker-vars
+  script:
+  - make publish
+  rules:
+  - if: '$CI_PIPELINE_SOURCE == "web"'
+    when: always

Dockerfile

@@ -0,0 +1,67 @@
+FROM alpine:3.11
+
+ARG ANSIBLE_UID=1200
+ARG ANSIBLE_USER=ansible
+ARG ANSIBLE_VERSION=2.9.7
+ARG PLEROMA_PLAYBOOK_DIRECTORY=/ansible/pleroma-playbook
+ARG BUILD_TIME
+ARG BUILD_TAG
+
+# Annotation (label) schema based on the OCI image specification.
+# https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.authors="Dan Anglin <d.n.i.anglin@gmail.com>" \
+      org.opencontainers.image.created=${BUILD_TIME} \
+      org.opencontainers.image.documentation="https://gitlab.com/dananglin/pleroma-ansible-playbook/-/blob/master/README.md" \
+      org.opencontainers.image.source="https://gitlab.com/dananglin/pleroma-ansible-playbook.git" \
+      org.opencontainers.image.version=${BUILD_TAG} \
+      org.opencontainers.image.vendor="Dan Anglin" \
+      org.opencontainers.image.licenses="MIT" \
+      org.opencontainers.image.title="Pleroma Ansible Playbook" \
+      org.opencontainers.image.description="Ansible playbook that installs, configures and customizes Pleroma on a Alpine host."
+
+# TODO: Remove sshpass when it is possible to do so.
+RUN \
+    apk add --no-cache \
+      ca-certificates=20191127-r2 \
+      make=4.2.1-r2 \
+      openssh-client=8.1_p1-r0 \
+      openssl=1.1.1g-r0 \
+      python3=3.8.2-r0 \
+      bash=5.0.11-r1 \
+      sshpass=1.06-r0 \
+    && \
+    apk add --no-cache --virtual .build-deps \
+      python3-dev=3.8.2-r0 \
+      libffi-dev=3.2.1-r6 \
+      openssl-dev=1.1.1g-r0 \
+      build-base=0.5-r1 \
+    && \
+    pip3 install --upgrade \
+      pip==20.0.2 \
+      cffi==1.14.0 \
+    && \
+    pip install \
+      ansible==${ANSIBLE_VERSION} \
+      hvac==0.10.3 \
+    && \
+    apk del \
+      .build-deps \
+    && \
+    adduser -u ${ANSIBLE_UID} -s /bin/sh -D ${ANSIBLE_USER}
+
+COPY --chown=${ANSIBLE_UID}:${ANSIBLE_UID} library ${PLEROMA_PLAYBOOK_DIRECTORY}/library/
+COPY --chown=${ANSIBLE_UID}:${ANSIBLE_UID} roles ${PLEROMA_PLAYBOOK_DIRECTORY}/roles/
+COPY --chown=${ANSIBLE_UID}:${ANSIBLE_UID} playbook.yml ${PLEROMA_PLAYBOOK_DIRECTORY}/playbook.yml
+COPY --chown=${ANSIBLE_UID}:${ANSIBLE_UID} Makefile ${PLEROMA_PLAYBOOK_DIRECTORY}/Makefile
+
+ENV ANSIBLE_HOST_KEY_CHECKING=False \
+    ANSIBLE_PYTHON_INTERPRETER=/usr/bin/python3 \
+    ANSIBLE_SSH_PIPELINING=False \
+    ANSIBLE_GATHERING=smart \
+    ANSIBLE_RETRY_FILES_ENABLED=False \
+    ANSIBLE_PERSISTENT_CONNECT_TIMEOUT=30 \
+    ANSIBLE_PERSISTENT_COMMAND_TIMEOUT=60
+
+USER ${ANSIBLE_USER}
+WORKDIR ${PLEROMA_PLAYBOOK_DIRECTORY}
+CMD [ "make", "pleroma" ]

Makefile

@@ -1,6 +1,15 @@
 VAPID_PRIVATE_KEY_FILE := vapid-private-key.pem
+INVENTORY ?= "hosts.yml"
+PLAYBOOK_TAGS ?= "all"
+PLAYBOOK_FILE ?= "playbook.yml"
+DOCKERFILE ?= Dockerfile
+DOCKER_CONTEXT ?= .
+IMAGE_NAME ?= pleroma-ansible-playbook
+IMAGE_TAG ?= latest
+IMAGE_DATE ?= nil
+DOCKER_IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
 
-PHONY: secret_key_base signing_salt vapid_private_key vapid_public_key
+PHONY: secret_key_base signing_salt vapid_private_key vapid_public_key test_modules_unit image publish pleroma
 
 all: secret_key_base signing_salt vapid_key_pair
 
@@ -27,3 +36,12 @@ vapid_public_key: $(VAPID_PRIVATE_KEY_FILE)
 
 test_modules_unit:
 	@find ./library -mindepth 1 -maxdepth 1 -type f -name test_*.py | xargs python3
+
+image:
+	@docker build $(BUILD_ARGS) -f $(DOCKERFILE) -t $(DOCKER_IMAGE) $(DOCKER_CONTEXT)
+
+publish: image
+	@docker push $(DOCKER_IMAGE)
+
+pleroma:
+	ansible-playbook --inventory $(INVENTORY) --tags $(PLAYBOOK_TAGS) $(EXTRA_ARGS) $(PLAYBOOK_FILE)

README.md

@@ -1,30 +1,59 @@
 # Ansible Playbook for Pleroma
 
+## Table of content
+
+- [Summary](#summary)
+- [Requirements](#requirements)
+- [Configuration](#configuration)
+- [Secrets](#secrets)
+- [Guide to setting up and running the playbook](#guide-to-setting-up-and-running-the-playbook)
+
 ## Summary
 
-This project was inspired by the official [Pleroma OTP installation guide](https://docs.pleroma.social/otp_en.html#content)
-and contains a playbook which installs and configures Pleroma on a single Alpine Linux host.
-It currently contains four roles, including:
+This project is a configurable playbook that can install, configure and customise Pleroma on an Alpine Linux host.
+The playbook is currently used to manage my personal instance at https://fedi.dananglin.me.uk.
+It currently only supports installing Pleroma on a single host but will support installing it across multipe hosts in the future.
 
-- **init:** merges the default configuration with the user's custom configuration.
-- **pleroma-postgres:** installs and configures the PostgreSQL database.
-- **pleroma-main:** installs/upgrades Pleroma and configures both the backend and frontend.
-- **pleroma-nginx:** installs and configures Nginx, creates SSL certificates using Let's Encrypt and adds support for proving your Pleroma site with Keybase.
+This project was inspired by the official [Pleroma OTP installation guide](https://docs.pleroma.social/otp_en.html#content).
 
-This project is currently used to manage my personal instance at https://fedi.dananglin.me.uk.
+This project is developed over at https://gitlab.com/dananglin/pleroma-ansible-playbook. The master branch is mirrored over at https://github.com/dananglin/pleroma-ansible-playbook.
 
-## Additional Features
+The project's icon ([Books icon](https://icons8.com/icon/114010/books)) was downloaded from [Icons8](https://icons8.com).
 
-- **Let's Encrypt support:** This playbook creates a SSL certificate using Let's Encrypt.
+### Ansible roles
+
+There are four roles used to install and configure your Pleroma instance:
+
+- **init:** merges the default configuration with your custom configuration.
+- **pleroma-database:** maintains the database layer by installing and configuring the PostgreSQL database server, creating the database user and creating and administrating the Pleroma database.
+- **pleroma-main:** maintains the main layer by handling the Pleroma installation, Pleroma upgrades and the configuration of the Pleroma frontend and backend.
+- **pleroma-proxy:** maintains the proxy layer by installing and configuring Nginx, creating the TLS certificates using Let's Encrypt, adding support for proving your Pleroma site with Keybase, etc.
+
+### Additional features
+
+- **Let's Encrypt support:** This playbook creates a TLS certificate using Let's Encrypt.
 - **Keybase support:** Pleroma does not support Keybase out of the box but you can still prove that your ownership of your Pleroma site.
-- **Custom default background:** Specify an image to use as the default background of your Pleroma site.
+- **Set default background:** You can specify an image to use as the default background of your Pleroma site.
+- **Upload custom themes:** You can upload custom Pleroma themes in to your Pleroma instance.
+- **Set default theme:** You can specify the default Pleroma theme.
 
 ## Requirements
 
-- A controller host running [Ansible](https://www.ansible.com/) version 2.8+.
-- `make` and `openssl` on the controller host which are used to generate secret values.
-- A target host running [Alpine Linux](https://www.alpinelinux.org/) version 3.10.
+### For your controller host
+
+- The controller host should the following packages installed:
+  - [Ansible](https://www.ansible.com/) version 2.8+.
+  - make
+  - openssl
+
+### For your target Pleroma host
+
+- The target host should be running [Alpine Linux](https://www.alpinelinux.org/) version 3.10+.
 - A (sub)domain which resolves to the IP address of the target host.
+- A user with sudo privileges (optional, but preferable).
+- The following packages installed:
+  - sudo
+  - python (version 3.5 or higher)
 
 ## Configuration
 
@@ -44,7 +73,7 @@ Following secrets are not included in the default configuration and must be gene
 - **vapid key pair for web push encryption:** This is a private and public key pair so that Pleroma can used [VAPID](https://tools.ietf.org/html/rfc8292) to identify itself to the web push service (for notifications in the browser).
 - **database password:** This is used to authenticate access to the Pleroma database.
 
-Insstructions on generating these can be found in the guide below.
+Instructions on generating these can be found in the guide below.
 It is recommended to encrypt these secrets using [Ansible Vault](https://docs.ansible.com/ansible/latest/user_guide/vault.html).
 
 ## Guide to setting up and running the playbook
@@ -54,7 +83,7 @@ It is recommended to encrypt these secrets using [Ansible Vault](https://docs.an
   $ cp examples/inventory.yml ./
   ```
 
-- In the inventory file you've just copied change **\<ANSIBLE\_HOSTS\>** to the IP address of the target host and change **\<ANSIBLE\_USER\>** to the user on the target host with sudo priviledges.
+- In the inventory file you've just copied change **\<ANSIBLE\_HOSTS\>** to the IP address of the target host and change **\<ANSIBLE\_USER\>** to the user on the target host with sudo privileges.
 
 - Copy the [example playbook file](examples/site.yml) to the root of the project.
   ```bash

examples/site.yml

@@ -11,9 +11,9 @@
   become: yes
   become_method: sudo
   roles:
-  - pleroma-postgres
+  - pleroma-database
   tags:
-  - pleroma-postgres
+  - pleroma-database
 
 - name: Installing and configuring the Pleroma backend.
   hosts: pleroma_backend
@@ -29,6 +29,6 @@
   become: yes
   become_method: sudo
   roles:
-  - pleroma-nginx
+  - pleroma-proxy
   tags:
-  - pleroma-nginx
+  - pleroma-proxy

playbook.yml

@@ -0,0 +1,34 @@
+---
+- name: Initialising the playbook.
+  hosts: all
+  roles:
+  - init
+  tags:
+  - always
+
+- name: Setting up the Pleroma database.
+  hosts: pleroma_database
+  become: yes
+  become_method: sudo
+  roles:
+  - pleroma-database
+  tags:
+  - pleroma-database
+
+- name: Setting up Pleroma.
+  hosts: pleroma_main
+  become: yes
+  become_method: sudo
+  roles:
+  - pleroma-main
+  tags:
+  - pleroma-main
+
+- name: Setting up the Pleroma proxy layer.
+  hosts: pleroma_proxy
+  become: yes
+  become_method: sudo
+  roles:
+  - pleroma-proxy
+  tags:
+  - pleroma-proxy

roles/init/defaults/main.yml

@@ -2,15 +2,15 @@
 pleroma_defaults:
   config:
     email: admin@localhost
-    notifyEmail: no_reply@{{ pleroma.config.host }}
+    notifyEmail: no_reply@pleroma.localhost
     host: pleroma.localhost
     listeningPort: 4000
     instanceName: "Ansible Pleroma"
     instanceDescription: "A new single instance of Pleroma"
     characterLimit: 5000
     registrationsOpen: "false"
-    dynamicConfiguration: "false"
-    healthCheck: "true"
+    configurableFromDatabase: "false"
+    healthCheck: "false"
     remotePostRetentionDays: 90
     logLevel: info
     federating: "true"

roles/pleroma-main/tasks/main.yml

@@ -79,7 +79,7 @@
   when: enable_pleroma_download | default(False)
 
 - name: Registering the installed version of Pleroma.
-  shell: "{{ pleroma_user.home }}/bin/pleroma version | awk '{print $2}'"
+  shell: "{{ pleroma_user.home }}/bin/pleroma version | awk '{print $2}' | awk -F - '{print $1}'"
   register: pleroma_installed_version
   when: enable_pleroma_upgrade | default(False)
 
@@ -89,7 +89,7 @@
   when: enable_pleroma_upgrade | default(False)
 
 - name: Registering the downloaded version of Pleroma.
-  shell: /tmp/release/bin/pleroma version | awk '{print $2}'
+  shell: "/tmp/release/bin/pleroma version | awk '{print $2}' | awk -F - '{print $1}'"
   register: pleroma_downloaded_version
   when: enable_pleroma_upgrade | default(False)
 

roles/pleroma-main/templates/etc_pleroma_config.exs.j2

@@ -23,7 +23,7 @@ config :pleroma, :instance,
   registrations_open: {{ pleroma.config.registrationsOpen }},
   healthcheck: {{ pleroma.config.healthCheck }},
   remote_post_retention_days: {{ pleroma.config.remotePostRetentionDays }},
-  dynamic_configuration: {{ pleroma.config.dynamicConfiguration }},
+  configurable_from_database: {{ pleroma.config.configurableFromDatabase }},
   federating: {{ pleroma.config.federating }},
   rewrite_policy: [Pleroma.Web.ActivityPub.MRF.SimplePolicy]
 

roles/pleroma-main/templates/var_lib_pleroma_static_static_styles.json.j2

@@ -1,6 +1,7 @@
 {
-  "pleroma-dark": [ "Pleroma Dark", "#121a24", "#182230", "#b9b9ba", "#d8a070", "#d31014", "#0fa00f", "#0095ff", "#ffa500" ],
-  "pleroma-light": [ "Pleroma Light", "#f2f4f6", "#dbe0e8", "#304055", "#f86f0f", "#d31014", "#0fa00f", "#0095ff", "#ffa500" ],
+
+  "pleroma-dark": "/static/themes/pleroma-dark.json",
+  "pleroma-light": "/static/themes/pleroma-light.json",
   "pleroma-amoled": [ "Pleroma Dark AMOLED", "#000000", "#111111", "#b0b0b1", "#d8a070", "#aa0000", "#0fa00f", "#0095ff", "#d59500"],
   "classic-dark": [ "Classic Dark", "#161c20", "#282e32", "#b9b9b9", "#baaa9c", "#d31014", "#0fa00f", "#0095ff", "#ffa500" ],
   "bird": [ "Bird", "#f8fafd", "#e6ecf0", "#14171a", "#0084b8", "#e0245e", "#17bf63", "#1b95e0", "#fab81e"],
@@ -12,7 +13,8 @@
   "redmond-xxi": "/static/themes/redmond-xxi.json",
   "breezy-dark": "/static/themes/breezy-dark.json",
   "breezy-light": "/static/themes/breezy-light.json",
-  "mammal": "/static/themes/mammal.json"
+  "mammal": "/static/themes/mammal.json",
+  "paper": "/static/themes/paper.json"
 
 {% for i in pleroma.config.frontend.themes.custom %}
   , "{{ i.name }}": "/static/themes/{{ i.name }}.json"

roles/pleroma-proxy/templates/etc_ngnix_confd_pleroma.conf.j2

@@ -64,6 +64,7 @@ server {
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
         proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
         # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
         # and `localhost.` resolves to [::0] on some systems: see issue #930
@@ -83,9 +84,7 @@ server {
         proxy_ignore_client_abort on;
         proxy_buffering           on;
         chunked_transfer_encoding on;
-        proxy_ignore_headers Cache-Control;
-        proxy_hide_header Cache-Control;
-        proxy_pass http://localhost:{{ pleroma.config.listeningPort }};
+        proxy_pass                http://127.0.0.1:{{ pleroma.config.listeningPort }};
     }
 
     {% if pleroma.keybase.enable == true -%}

test/pleroma_test_env/.dockerignore

@@ -0,0 +1,2 @@
+*
+!files

test/pleroma_test_env/Dockerfile

@@ -0,0 +1,30 @@
+FROM dockage/alpine:3.10-openrc
+
+ARG TEST_ENV_UID=1100
+ARG TEST_ENV_USER=admin
+ARG TEST_ENV_PASSWORD
+ARG SSH_DIR=/home/admin/.ssh
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
+
+RUN apk add --no-cache \
+        bash=5.0.0-r0 \
+        openssh=8.1_p1-r0 \
+        sudo=1.8.27-r2 \
+        python3=3.7.7-r0 \
+    && \
+    adduser -u ${TEST_ENV_UID} -s /bin/bash -D ${TEST_ENV_USER} && \
+    echo ${TEST_ENV_USER}:${TEST_ENV_PASSWORD} | chpasswd && \
+    mkdir ${SSH_DIR} && \
+    chown ${TEST_ENV_USER}:${TEST_ENV_USER} ${SSH_DIR} && chmod 0700 ${SSH_DIR} && \
+    touch ${SSH_DIR}/authorized_keys && \
+    chown ${TEST_ENV_USER}:${TEST_ENV_USER} ${SSH_DIR}/authorized_keys && \
+    chmod 0600 ${SSH_DIR}/authorized_keys && \
+    rc-status && \
+    touch /run/openrc/softlevel
+
+COPY files/entrypoint /entrypoint
+COPY files/admin-sudoers /etc/sudoers.d/admin
+
+EXPOSE 22 80 443
+
+CMD ["/entrypoint"]

test/pleroma_test_env/files/admin-sudoers

@@ -0,0 +1 @@
+admin ALL=(ALL) NOPASSWD: ALL

test/pleroma_test_env/files/entrypoint

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set euo -pipefail
+
+# Add authorized keys
+if ! [ -z "${AUTHORIZED_KEYS}" ]; then 
+    echo ${AUTHORIZED_KEYS} > /home/admin/.ssh/authorized_keys
+fi
+
+# Start SSH service
+rc-service sshd start
+
+# Sleep and wait
+sleep 30d