flow/services
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Activity

refactor: update project structure

Browse source 
Changes:

- Most of the go code is now located in internal packages.
- Code refactored and simplified in some cases.
- Removed the 'download' mage target and integrated the download code
  into the internal 'prepare' package.
- Moved all mage target code to magefile.go.
- Added missing descriptions to the mage targets.
- Updated go.mod.

Fixed:

- Created a custom function to validate the checksum of the
  downloaded Woodpecker tar file.
- Specified the environment when running 'clean'.
This commit is contained in:
Dan Anglin 2023-11-24 09:56:35 +00:00
parent beb3826190
commit 25325e2856
Signed by: dananglin
GPG key ID: 0C1D44CFBEE68638
20 changed files with 915 additions and 744 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -1,5 +1,5 @@
module flow/services
go 1.20
go 1.21
require github.com/magefile/mage v1.15.0

121
magefiles/config.go → internal/config/config.go
View file

@ -1,6 +1,4 @@
//go:build mage
package main
package config
import (
"encoding/json"
@ -9,28 +7,33 @@ import (
"path/filepath"
)
type config struct {
RootDomain string `json:"rootDomain"`
FlowGID int32 `json:"flowGID"`
Docker dockerConfig `json:"docker"`
Traefik traefikConfig `json:"traefik"`
Forgejo forgejoConfig `json:"forgejo"`
GoToSocial gotosocialConfig `json:"gotosocial"`
Woodpecker woodpeckerConfig `json:"woodpecker"`
Landing landingConfig `json:"landing"`
const (
configDir string = "./config/"
configFileName string = "services.json"
)
type Config struct {
RootDomain string `json:"rootDomain"`
FlowGID int32 `json:"flowGID"`
Docker Docker `json:"docker"`
Traefik Traefik `json:"traefik"`
Forgejo Forgejo `json:"forgejo"`
GoToSocial Gotosocial `json:"gotosocial"`
Woodpecker Woodpecker `json:"woodpecker"`
Landing LandingPage `json:"landing"`
}
type dockerConfig struct {
Host string `json:"host"`
Network dockerNetworkConfig `json:"network"`
type Docker struct {
Host string `json:"host"`
Network DockerNetwork `json:"network"`
}
type dockerNetworkConfig struct {
type DockerNetwork struct {
Name string `json:"name"`
Subnet string `json:"subnet"`
}
type traefikConfig struct {
type Traefik struct {
Version string `json:"version"`
CheckNewVersion bool `json:"checkNewVersion"`
ExternalSSHPort int32 `json:"externalSSHPort"`
@ -47,42 +50,42 @@ type traefikConfig struct {
DynamicConfigDirectory string `json:"dynamicConfigDirectory"`
}
type forgejoConfig struct {
Version string `json:"version"`
Name string `json:"name"`
Subdomain string `json:"subdomain"`
ContainerName string `json:"containerName"`
ContainerIpv4Address string `json:"containerIpv4Address"`
SshPort int32 `json:"sshPort"`
HttpPort int32 `json:"httpPort"`
RunMode string `json:"runMode"`
LogLevel string `json:"logLevel"`
LinuxUID int32 `json:"linuxUID"`
DataHostDirectory string `json:"dataHostDirectory"`
DataContainerDirectory string `json:"dataContainerDirectory"`
Home string `json:"home"`
Work string `json:"work"`
Custom string `json:"custom"`
AppIni string `json:"appIni"`
Bin string `json:"bin"`
Tmp string `json:"tmp"`
SecretHostDirectory string `json:"secretHostDirectory"`
SecretContainerDirectory string `json:"secretContainerDirectory"`
SecretKey string `json:"secretKey"`
InternalToken string `json:"internalToken"`
LfsJwtSecret string `json:"lfsJwtSecret"`
Oauth2Enable bool `json:"oauth2Enable"`
Oauth2JwtSigningAlgo string `json:"oauth2JwtSigningAlgo"`
Oauth2JwtSecret string `json:"oauth2JwtSecret"`
Actions forgejoActionsConfig `json:"actions"`
type Forgejo struct {
Version string `json:"version"`
Name string `json:"name"`
Subdomain string `json:"subdomain"`
ContainerName string `json:"containerName"`
ContainerIpv4Address string `json:"containerIpv4Address"`
SshPort int32 `json:"sshPort"`
HttpPort int32 `json:"httpPort"`
RunMode string `json:"runMode"`
LogLevel string `json:"logLevel"`
LinuxUID int32 `json:"linuxUID"`
DataHostDirectory string `json:"dataHostDirectory"`
DataContainerDirectory string `json:"dataContainerDirectory"`
Home string `json:"home"`
Work string `json:"work"`
Custom string `json:"custom"`
AppIni string `json:"appIni"`
Bin string `json:"bin"`
Tmp string `json:"tmp"`
SecretHostDirectory string `json:"secretHostDirectory"`
SecretContainerDirectory string `json:"secretContainerDirectory"`
SecretKey string `json:"secretKey"`
InternalToken string `json:"internalToken"`
LfsJwtSecret string `json:"lfsJwtSecret"`
Oauth2Enable bool `json:"oauth2Enable"`
Oauth2JwtSigningAlgo string `json:"oauth2JwtSigningAlgo"`
Oauth2JwtSecret string `json:"oauth2JwtSecret"`
Actions ForgejoActions `json:"actions"`
}
type forgejoActionsConfig struct {
type ForgejoActions struct {
Enabled bool `json:"enabled"`
DefaultActionsURL string `json:"defaultActionsURL"`
}
type gotosocialConfig struct {
type Gotosocial struct {
Version string `json:"version"`
Name string `json:"name"`
LogLevel string `json:"logLevel"`
@ -100,7 +103,7 @@ type gotosocialConfig struct {
TZ string `json:"tz"`
}
type woodpeckerConfig struct {
type Woodpecker struct {
Version string `json:"version"`
LogLevel string `json:"logLevel"`
LinuxUID int32 `json:"linuxUID"`
@ -119,24 +122,24 @@ type woodpeckerConfig struct {
ForgejoClientSecret string `json:"forgejoClientSecret"`
}
type landingConfig struct {
Version string `json:"version"`
ContainerName string `json:"containerName"`
ContainerIpv4Address string `json:"containerIpv4Address"`
Services []landingConfigLinks `json:"services"`
Profiles []landingConfigLinks `json:"profiles"`
Port int32 `json:"port"`
ImageDigest string `json:"imageDigest"`
type LandingPage struct {
Version string `json:"version"`
ContainerName string `json:"containerName"`
ContainerIpv4Address string `json:"containerIpv4Address"`
Services []LandingPageLinks `json:"services"`
Profiles []LandingPageLinks `json:"profiles"`
Port int32 `json:"port"`
ImageDigest string `json:"imageDigest"`
}
type landingConfigLinks struct {
type LandingPageLinks struct {
Title string `json:"title"`
URL string `json:"url"`
Rel string `json:"rel"`
}
func newConfig(environment string) (config, error) {
var c config
func NewConfig(environment string) (Config, error) {
var c Config
path := filepath.Join(configDir, environment, configFileName)

142
internal/download/download.go Normal file
View file

@ -0,0 +1,142 @@
package download
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"github.com/magefile/mage/sh"
)
type Bundle struct {
DestinationDir string
Packages []Pack
Checksum Checksum
ValidateGPGSignature bool
}
type Pack struct {
File Object
GPGSignature Object
}
type Checksum struct {
File Object
Validate bool
ValidateFunc func() error
}
type Object struct {
Source string
Destination string
}
// Download downloads all the files in the download pack,
// verifies all the GPG signatures (if enabled) and
// verifies the checksums (if enabled).
func Download(bundle Bundle) error {
if err := os.MkdirAll(bundle.DestinationDir, 0o750); err != nil {
return fmt.Errorf("unable to make '%s'; %w", bundle.DestinationDir, err)
}
var objects []Object
for i := range bundle.Packages {
objects = append(objects, bundle.Packages[i].File)
if bundle.ValidateGPGSignature {
objects = append(objects, bundle.Packages[i].GPGSignature)
}
}
if bundle.Checksum.Validate {
objects = append(objects, bundle.Checksum.File)
}
for _, object := range objects {
if err := func() error {
_, err := os.Stat(object.Destination)
if err == nil {
fmt.Printf("%s is already downloaded.\n", object.Destination)
return nil
}
file, err := os.Create(object.Destination)
if err != nil {
return fmt.Errorf("unable to create %s; %w", object.Destination, err)
}
defer file.Close()
client := http.Client{
CheckRedirect: func(r *http.Request, _ []*http.Request) error {
r.URL.Opaque = r.URL.Path
return nil
},
}
resp, err := client.Get(object.Source)
if err != nil {
return err
}
defer resp.Body.Close()
size, err := io.Copy(file, resp.Body)
if err != nil {
return err
}
fmt.Printf("Downloaded %s with size %d.\n", object.Destination, size)
return nil
}(); err != nil {
return err
}
}
if bundle.ValidateGPGSignature {
for i := range bundle.Packages {
if err := sh.RunV("gpg", "--verify", bundle.Packages[i].GPGSignature.Destination, bundle.Packages[i].File.Destination); err != nil {
return fmt.Errorf("GPG verification failed for '%s'; %w", bundle.Packages[i].File.Destination, err)
}
}
}
if bundle.Checksum.Validate {
var err error
if bundle.Checksum.ValidateFunc != nil {
err = bundle.Checksum.ValidateFunc()
} else {
err = validateChecksum(bundle.DestinationDir, bundle.Checksum.File.Destination)
}
if err != nil {
return fmt.Errorf("checksum validation failed; %w", err)
}
}
return nil
}
func validateChecksum(destinationDir, checksumPath string) error {
startDir, err := os.Getwd()
if err != nil {
return err
}
if err := os.Chdir(destinationDir); err != nil {
return err
}
checksum := filepath.Base(checksumPath)
if err := sh.RunV("sha256sum", "--check", "--ignore-missing", checksum); err != nil {
return err
}
if err := os.Chdir(startDir); err != nil {
return err
}
return nil
}

7
internal/internal.go Normal file
View file

@ -0,0 +1,7 @@
package internal
const (
RootBuildDir string = "./build"
RootTemplatesDir string = "./templates"
RootAssetsDir string = "./assets"
)

249
internal/prepare/prepare.go Normal file
View file

@ -0,0 +1,249 @@
package prepare
import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"text/template"
"flow/services/internal"
"flow/services/internal/config"
"flow/services/internal/services"
"flow/services/internal/services/forgejo"
"flow/services/internal/services/gotosocial"
"flow/services/internal/services/woodpecker"
)
const templateExtension string = ".gotmpl"
func Prepare(environment, service string) error {
cfg, err := config.NewConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
if err := prepareCompose(cfg, environment); err != nil {
return fmt.Errorf("unable to prepare the compose file; %w", err)
}
if service == "all" {
if err := prepareAllServices(cfg, environment); err != nil {
return fmt.Errorf("error preparing the services; %w", err)
}
} else {
if err := prepareService(cfg, environment, service); err != nil {
return fmt.Errorf("error preparing %q; %w", service, err)
}
}
return nil
}
func prepareCompose(cfg config.Config, environment string) error {
compose := "compose"
buildDir := filepath.Join(internal.RootBuildDir, environment, compose)
if _, err := os.Stat(buildDir); err != nil {
if err := os.MkdirAll(buildDir, 0o700); err != nil {
return fmt.Errorf("unable to make %s; %w", buildDir, err)
}
}
if err := render(cfg, environment, compose); err != nil {
return fmt.Errorf("an error occurred whilst rendering the templates; %w", err)
}
return nil
}
func prepareAllServices(cfg config.Config, environment string) error {
allServices := services.All()
for i := range allServices {
if err := prepareService(cfg, environment, allServices[i]); err != nil {
return fmt.Errorf("error preparing %q; %w", allServices[i], err)
}
}
return nil
}
func prepareService(cfg config.Config, environment, service string) error {
buildDir := filepath.Join(internal.RootBuildDir, environment, service)
if _, err := os.Stat(buildDir); err != nil {
if err := os.MkdirAll(buildDir, 0o700); err != nil {
return fmt.Errorf("unable to make %s; %w", buildDir, err)
}
}
if err := downloadServiceFiles(cfg, environment, service); err != nil {
return fmt.Errorf("error downloading service files for %q; %w", service, err)
}
if err := copyAssets(environment, service); err != nil {
return fmt.Errorf("unable to copy the assets for %s; %w", service, err)
}
if err := render(cfg, environment, service); err != nil {
return fmt.Errorf("an error occurred whilst rendering the templates; %w", err)
}
return nil
}
func downloadServiceFiles(cfg config.Config, environment, service string) error {
switch service {
case services.Forgejo:
obj := forgejo.NewForgejo(environment, cfg.Forgejo.Version)
if err := obj.Download(); err != nil {
return fmt.Errorf("error downloading the files for %q; %w", services.Forgejo, err)
}
case services.Gotosocial:
obj := gotosocial.NewGotosocial(environment, cfg.GoToSocial.Version)
if err := obj.Download(); err != nil {
return fmt.Errorf("error downloading the files for %q; %w", services.Gotosocial, err)
}
case services.Woodpecker:
obj := woodpecker.NewWoodpecker(environment, cfg.Woodpecker.Version)
if err := obj.Download(); err != nil {
return fmt.Errorf("error downloading the files for %q; %w", services.Woodpecker, err)
}
default:
fmt.Printf("There's no files to download for %q.\n", service)
}
return nil
}
func render(cfg config.Config, environment, component string) error {
buildDirName := filepath.Join(internal.RootBuildDir, environment, component)
templateDirName := filepath.Join(internal.RootTemplatesDir, component)
_, err := os.Stat(templateDirName)
if err != nil {
if os.IsNotExist(err) {
fmt.Printf("There's no template directory for %q.\n", component)
return nil
}
return err
}
files, err := os.ReadDir(templateDirName)
if err != nil {
return fmt.Errorf("unable to read files from %s; %w ", templateDirName, err)
}
funcMap := template.FuncMap{
"default": defaultValue,
}
for _, f := range files {
err := func() error {
templateFilename := f.Name()
if f.IsDir() || !strings.HasSuffix(templateFilename, templateExtension) {
return nil
}
outputFilename := strings.TrimSuffix(templateFilename, templateExtension)
outputPath := filepath.Join(buildDirName, outputFilename)
file, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("unable to create the file '%s'; %w", outputPath, err)
}
defer file.Close()
templatePath := filepath.Join(templateDirName, templateFilename)
tmpl, err := template.New(templateFilename).Funcs(funcMap).ParseFiles(templatePath)
if err != nil {
return fmt.Errorf("unable to create a new template value from '%s'; %w", templateFilename, err)
}
if err = tmpl.Execute(file, cfg); err != nil {
return fmt.Errorf("unable to render the template to '%s'; %w", outputPath, err)
}
return nil
}()
if err != nil {
return fmt.Errorf("an error occurred whilst rendering the templates for '%s'; %w", component, err)
}
}
return nil
}
func copyAssets(environment, service string) error {
assetsDirName := filepath.Join(internal.RootAssetsDir, service)
if _, err := os.Stat(assetsDirName); err != nil {
if os.IsNotExist(err) {
fmt.Printf("There's no assets directory for %q.\n", service)
return nil
}
return err
}
buildDirName := filepath.Join(internal.RootBuildDir, environment, service, "assets")
walkDirFunc := func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if path == assetsDirName {
return nil
}
buildAssetPath := filepath.Join(buildDirName, strings.TrimPrefix(path, assetsDirName))
if d.IsDir() {
if err := os.MkdirAll(buildAssetPath, 0o750); err != nil {
return fmt.Errorf("unable to make %s; %w", path, err)
}
return nil
}
source, err := os.Open(path)
if err != nil {
return err
}
defer source.Close()
dest, err := os.Create(buildAssetPath)
if err != nil {
return err
}
defer dest.Close()
if _, err := io.Copy(dest, source); err != nil {
return err
}
return nil
}
err := filepath.WalkDir(assetsDirName, walkDirFunc)
if err != nil {
return fmt.Errorf("error walking the path '%s'; %w", assetsDirName, err)
}
return nil
}
func defaultValue(value, defaultValue string) string {
if value == "" {
return defaultValue
}
return value
}

40
internal/services/forgejo/download.go Normal file
View file

@ -0,0 +1,40 @@
package forgejo
import (
"fmt"
"flow/services/internal/download"
)
func (f Forgejo) Download() error {
bundle := download.Bundle{
DestinationDir: f.destinationDir,
Packages: []download.Pack{
{
File: download.Object{
Source: f.binaryURL,
Destination: f.binaryPath,
},
GPGSignature: download.Object{
Source: f.signatureURL,
Destination: f.signaturePath,
},
},
},
ValidateGPGSignature: true,
Checksum: download.Checksum{
File: download.Object{
Source: f.checksumURL,
Destination: f.checksumPath,
},
Validate: true,
ValidateFunc: nil,
},
}
if err := download.Download(bundle); err != nil {
return fmt.Errorf("error downloading files for Forgejo; %w", err)
}
return nil
}

59
internal/services/forgejo/forgejo.go Normal file
View file

@ -0,0 +1,59 @@
package forgejo
import (
"fmt"
"path/filepath"
"flow/services/internal"
"flow/services/internal/services"
)
type Forgejo struct {
binaryURL string
binaryPath string
signatureURL string
signaturePath string
checksumURL string
checksumPath string
destinationDir string
}
func NewForgejo(environment, version string) Forgejo {
forgejoBinaryFileFormat := "forgejo-%s-linux-amd64"
forgejoDigestExtension := ".sha256"
forgejoSignatureExtension := ".asc"
destinationDir := filepath.Join(internal.RootBuildDir, environment, services.Forgejo)
binaryPath := filepath.Join(destinationDir, fmt.Sprintf(forgejoBinaryFileFormat, version))
forgejo := Forgejo{
destinationDir: destinationDir,
binaryURL: fmt.Sprintf(
"https://codeberg.org/forgejo/forgejo/releases/download/v%s/forgejo-%s-linux-amd64",
version,
version,
),
binaryPath: filepath.Join(
destinationDir,
fmt.Sprintf(forgejoBinaryFileFormat, version),
),
signatureURL: fmt.Sprintf(
"https://codeberg.org/forgejo/forgejo/releases/download/v%s/forgejo-%s-linux-amd64.asc",
version,
version,
),
signaturePath: binaryPath + forgejoSignatureExtension,
checksumURL: fmt.Sprintf(
"https://codeberg.org/forgejo/forgejo/releases/download/v%s/forgejo-%s-linux-amd64.sha256",
version,
version,
),
checksumPath: binaryPath + forgejoDigestExtension,
}
return forgejo
}

43
internal/services/gotosocial/download.go Normal file
View file

@ -0,0 +1,43 @@
package gotosocial
import (
"fmt"
"flow/services/internal/download"
)
// Download downloads and validates the files for GoToSocial.
func (g Gotosocial) Download() error {
bundle := download.Bundle{
DestinationDir: g.destinationDir,
Packages: []download.Pack{
{
File: download.Object{
Source: g.binaryTarURL,
Destination: g.binaryTarFilepath,
},
},
{
File: download.Object{
Source: g.webAssetsTarURL,
Destination: g.webAssetsFilepath,
},
},
},
ValidateGPGSignature: false,
Checksum: download.Checksum{
File: download.Object{
Source: g.checksumURL,
Destination: g.checksumPath,
},
Validate: true,
ValidateFunc: nil,
},
}
if err := download.Download(bundle); err != nil {
return fmt.Errorf("error downloading the files for Gotosocial; %w", err)
}
return nil
}

48
internal/services/gotosocial/gotosocial.go Normal file
View file

@ -0,0 +1,48 @@
package gotosocial
import (
"fmt"
"path/filepath"
"flow/services/internal"
"flow/services/internal/services"
)
type Gotosocial struct {
binaryTarURL string
binaryTarFilepath string
checksumURL string
checksumPath string
webAssetsTarURL string
webAssetsFilepath string
destinationDir string
}
func NewGotosocial(environment, version string) Gotosocial {
destinationDir := filepath.Join(internal.RootBuildDir, environment, services.Gotosocial)
gotosocial := Gotosocial{
destinationDir: destinationDir,
binaryTarURL: fmt.Sprintf(
"https://github.com/superseriousbusiness/gotosocial/releases/download/v%s/gotosocial_%s_linux_amd64.tar.gz",
version,
version,
),
binaryTarFilepath: filepath.Join(destinationDir, fmt.Sprintf("gotosocial_%s_linux_amd64.tar.gz", version)),
webAssetsTarURL: fmt.Sprintf(
"https://github.com/superseriousbusiness/gotosocial/releases/download/v%s/gotosocial_%s_web-assets.tar.gz",
version,
version,
),
webAssetsFilepath: filepath.Join(destinationDir, fmt.Sprintf("gotosocial_%s_web-assets.tar.gz", version)),
checksumURL: fmt.Sprintf(
"https://github.com/superseriousbusiness/gotosocial/releases/download/v%s/checksums.txt",
version,
),
checksumPath: filepath.Join(destinationDir, fmt.Sprintf("gotosocial_%s_checksums.txt", version)),
}
return gotosocial
}

19
internal/services/services.go Normal file
View file

@ -0,0 +1,19 @@
package services
const (
Traefik string = "traefik"
Forgejo string = "forgejo"
Gotosocial string = "gotosocial"
Woodpecker string = "woodpecker"
Landing string = "landing"
)
func All() []string {
return []string{
Traefik,
Forgejo,
Gotosocial,
Woodpecker,
Landing,
}
}

85
internal/services/woodpecker/download.go Normal file
View file

@ -0,0 +1,85 @@
package woodpecker
import (
"bufio"
"crypto/sha256"
"fmt"
"io"
"os"
"strings"
"flow/services/internal/download"
)
func (w Woodpecker) Download() error {
bundle := download.Bundle{
DestinationDir: w.destinationDir,
Packages: []download.Pack{
{
File: download.Object{
Source: w.binaryTarURL,
Destination: w.binaryTarPath,
},
},
},
ValidateGPGSignature: false,
Checksum: download.Checksum{
File: download.Object{
Source: w.checksumURL,
Destination: w.checksumPath,
},
Validate: true,
ValidateFunc: w.checksumValidation,
},
}
if err := download.Download(bundle); err != nil {
return err
}
return nil
}
func (w Woodpecker) checksumValidation() error {
var wantChecksum, gotChecksum string
// get the tar file's checksum
tarFile, err := os.Open(w.binaryTarPath)
if err != nil {
return fmt.Errorf("unable to open %s; %w", w.binaryTarPath, err)
}
defer tarFile.Close()
h := sha256.New()
if _, err = io.Copy(h, tarFile); err != nil {
return fmt.Errorf("unable to get the shasum for %s; %w", w.binaryTarPath, err)
}
gotChecksum = fmt.Sprintf("%x", h.Sum(nil))
// get the expected checksum from the checksum file
checksumFile, err := os.Open(w.checksumPath)
if err != nil {
return fmt.Errorf("unable to open %s; %w", w.checksumPath, err)
}
defer checksumFile.Close()
scanner := bufio.NewScanner(checksumFile)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "woodpecker-server_linux_amd64.tar.gz") {
wantChecksum = strings.Split(line, " ")[0]
break
}
}
// compare the two checksum strings
if gotChecksum != wantChecksum {
return fmt.Errorf("checksum validation failed: want %s, got %s", wantChecksum, gotChecksum)
} else {
fmt.Printf("Checksum validation successful: %s\n", gotChecksum)
}
return nil
}

43
internal/services/woodpecker/woodpecker.go Normal file
View file

@ -0,0 +1,43 @@
package woodpecker
import (
"fmt"
"path/filepath"
"flow/services/internal"
"flow/services/internal/services"
)
type Woodpecker struct {
binaryTarURL string
binaryTarPath string
checksumURL string
checksumPath string
destinationDir string
}
func NewWoodpecker(environment, version string) Woodpecker {
destinationDir := filepath.Join(internal.RootBuildDir, environment, services.Woodpecker)
woodpecker := Woodpecker{
destinationDir: destinationDir,
binaryTarURL: fmt.Sprintf(
"https://github.com/woodpecker-ci/woodpecker/releases/download/v%s/woodpecker-server_linux_amd64.tar.gz",
version,
),
binaryTarPath: filepath.Join(
destinationDir,
fmt.Sprintf("woodpecker-server-%s_linux_amd64.tar.gz", version),
),
checksumURL: fmt.Sprintf(
"https://github.com/woodpecker-ci/woodpecker/releases/download/v%s/checksums.txt",
version,
),
checksumPath: filepath.Join(
destinationDir,
fmt.Sprintf("woodpecker_%s_checksums.txt", version),
),
}
return woodpecker
}

31
magefiles/clean.go
View file

@ -1,31 +0,0 @@
//go:build mage
package main
import (
"os"
"github.com/magefile/mage/sh"
)
// Clean cleans the workspace.
func Clean() error {
buildDir := "./build"
objects, err := os.ReadDir(buildDir)
if err != nil {
return err
}
for i := range objects {
name := objects[i].Name()
if name != ".gitkeep" {
if err := sh.Rm(buildDir + "/" + name); err != nil {
return err
}
}
}
return nil
}

12
magefiles/const.go
View file

@ -1,12 +0,0 @@
//go:build mage
package main
const (
configDir string = "./config/"
configFileName string = "services.json"
rootBuildDir string = "./build"
templateExtension string = ".gotmpl"
rootTemplatesDir string = "./templates"
rootAssetsDir string = "./assets"
)

41
magefiles/deploy.go
View file

@ -1,41 +0,0 @@
//go:build mage
package main
import (
"fmt"
"os"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
// Deploy deploys the services to the Flow Platform.
func Deploy(environment, name string) error {
mg.Deps(
mg.F(Prepare, environment, name),
)
cfg, err := newConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
os.Setenv("DOCKER_HOST", cfg.Docker.Host)
command := []string{
"docker",
"compose",
"--project-directory",
fmt.Sprintf("%s/%s/compose", rootBuildDir, environment),
"up",
"-d",
"--build",
}
if name != "all" {
command = append(command, name)
}
return sh.RunV(command[0], command[1:]...)
}

322
magefiles/download.go
View file

@ -1,322 +0,0 @@
//go:build mage
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"github.com/magefile/mage/sh"
)
// Download downloads the binaries for a given service.
func Download(environment, name string) error {
cfg, err := newConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %v", err)
}
switch name {
case "forgejo":
if err := downloadForgejo(environment, cfg.Forgejo.Version); err != nil {
return fmt.Errorf("an error occurred whilst getting the forgejo binary; %w", err)
}
case "gotosocial":
if err := downloadGoToSocial(environment, cfg.GoToSocial.Version); err != nil {
return fmt.Errorf("an error occurred whilst getting the packages for GoToSocial; %w", err)
}
case "woodpecker":
if err := downloadWoodpecker(environment, cfg.Woodpecker.Version); err != nil {
return fmt.Errorf("an error occurred whilst getting the packages for Woodpecker; %w", err)
}
default:
fmt.Printf("There's no files to download for %q.\n", name)
}
return nil
}
// downloadWoodpecker downloads and validates the files for the Woodpecker deployment.
func downloadWoodpecker(environment, version string) error {
destinationDir := filepath.Join(rootBuildDir, environment, "woodpecker")
binaryTarUrl := fmt.Sprintf(
"https://github.com/woodpecker-ci/woodpecker/releases/download/v%s/woodpecker-server_linux_amd64.tar.gz",
version,
)
binaryTarFilepath := filepath.Join(
destinationDir,
fmt.Sprintf("woodpecker-server-%s_linux_amd64.tar.gz", version),
)
checksumUrl := fmt.Sprintf(
"https://github.com/woodpecker-ci/woodpecker/releases/download/v%s/checksums.txt",
version,
)
checksumFilePath := filepath.Join(
destinationDir,
fmt.Sprintf("woodpecker_%s_checksums.txt", version),
)
pack := downloadPack{
destinationDir: destinationDir,
packages: []pack{
{
file: object{
source: binaryTarUrl,
destination: binaryTarFilepath,
},
},
},
validateGPGSignature: false,
checksum: object{
source: checksumUrl,
destination: checksumFilePath,
},
validateChecksum: false,
}
if err := download(pack); err != nil {
return err
}
return nil
}
// downloadForgejo downloads and validates the Forgejo files.
func downloadForgejo(environment, version string) error {
var (
forgejoBinaryFileFormat = "forgejo-%s-linux-amd64"
forgejoDigestExtension = ".sha256"
forgejoSignatureExtension = ".asc"
)
destinationDir := filepath.Join(rootBuildDir, environment, "forgejo")
binaryUrl := fmt.Sprintf(
"https://codeberg.org/forgejo/forgejo/releases/download/v%s/forgejo-%s-linux-amd64",
version,
version,
)
binaryPath := filepath.Join(
destinationDir,
fmt.Sprintf(forgejoBinaryFileFormat, version),
)
signatureUrl := fmt.Sprintf(
"https://codeberg.org/forgejo/forgejo/releases/download/v%s/forgejo-%s-linux-amd64.asc",
version,
version,
)
signaturePath := binaryPath + forgejoSignatureExtension
checksumUrl := fmt.Sprintf(
"https://codeberg.org/forgejo/forgejo/releases/download/v%s/forgejo-%s-linux-amd64.sha256",
version,
version,
)
checksumPath := binaryPath + forgejoDigestExtension
pack := downloadPack{
destinationDir: destinationDir,
packages: []pack{
{
file: object{
source: binaryUrl,
destination: binaryPath,
},
gpgSignature: object{
source: signatureUrl,
destination: signaturePath,
},
},
},
validateGPGSignature: true,
checksum: object{
source: checksumUrl,
destination: checksumPath,
},
validateChecksum: true,
}
if err := download(pack); err != nil {
return err
}
return nil
}
// downloadGoToSocial downloads and validates the files for GoToSocial.
func downloadGoToSocial(environment, version string) error {
destinationDir := filepath.Join(rootBuildDir, environment, "gotosocial")
binaryTarUrl := fmt.Sprintf(
"https://github.com/superseriousbusiness/gotosocial/releases/download/v%s/gotosocial_%s_linux_amd64.tar.gz",
version,
version,
)
binaryTarFilepath := filepath.Join(destinationDir, fmt.Sprintf("gotosocial_%s_linux_amd64.tar.gz", version))
webAssetsTarUrl := fmt.Sprintf(
"https://github.com/superseriousbusiness/gotosocial/releases/download/v%s/gotosocial_%s_web-assets.tar.gz",
version,
version,
)
webAssetsFilepath := filepath.Join(destinationDir, fmt.Sprintf("gotosocial_%s_web-assets.tar.gz", version))
checksumUrl := fmt.Sprintf(
"https://github.com/superseriousbusiness/gotosocial/releases/download/v%s/checksums.txt",
version,
)
checksumFilePath := filepath.Join(destinationDir, fmt.Sprintf("gotosocial_%s_checksums.txt", version))
pack := downloadPack{
destinationDir: destinationDir,
packages: []pack{
{
file: object{
source: binaryTarUrl,
destination: binaryTarFilepath,
},
},
{
file: object{
source: webAssetsTarUrl,
destination: webAssetsFilepath,
},
},
},
validateGPGSignature: false,
checksum: object{
source: checksumUrl,
destination: checksumFilePath,
},
validateChecksum: true,
}
if err := download(pack); err != nil {
return err
}
return nil
}
type downloadPack struct {
destinationDir string
packages []pack
checksum object
validateGPGSignature bool
validateChecksum bool
}
type pack struct {
file object
gpgSignature object
}
type object struct {
source string
destination string
}
// download downloads all the files in the download pack,
// verifies all the GPG signatures (if enabled) and
// verifies the checksums (if enabled).
func download(pack downloadPack) error {
if err := os.MkdirAll(pack.destinationDir, 0o750); err != nil {
return fmt.Errorf("unable to make '%s'; %w", pack.destinationDir, err)
}
var objects []object
for i := range pack.packages {
objects = append(objects, pack.packages[i].file)
if pack.validateGPGSignature {
objects = append(objects, pack.packages[i].gpgSignature)
}
}
if pack.validateChecksum {
objects = append(objects, pack.checksum)
}
for _, object := range objects {
if err := func() error {
_, err := os.Stat(object.destination)
if err == nil {
fmt.Printf("%s is already downloaded.\n", object.destination)
return nil
}
file, err := os.Create(object.destination)
if err != nil {
return fmt.Errorf("unable to create %s; %w", object.destination, err)
}
defer file.Close()
client := http.Client{
CheckRedirect: func(r *http.Request, _ []*http.Request) error {
r.URL.Opaque = r.URL.Path
return nil
},
}
resp, err := client.Get(object.source)
if err != nil {
return err
}
defer resp.Body.Close()
size, err := io.Copy(file, resp.Body)
if err != nil {
return err
}
fmt.Printf("Downloaded %s with size %d.\n", object.destination, size)
return nil
}(); err != nil {
return err
}
}
if pack.validateGPGSignature {
for i := range pack.packages {
if err := sh.RunV("gpg", "--verify", pack.packages[i].gpgSignature.destination, pack.packages[i].file.destination); err != nil {
return fmt.Errorf("GPG verification failed for '%s'; %w", pack.packages[i].file.destination, err)
}
}
}
if pack.validateChecksum {
startDir, err := os.Getwd()
if err != nil {
return err
}
if err := os.Chdir(pack.destinationDir); err != nil {
return err
}
checksum := filepath.Base(pack.checksum.destination)
if err := sh.RunV("sha256sum", "--check", "--ignore-missing", checksum); err != nil {
return err
}
if err := os.Chdir(startDir); err != nil {
return err
}
}
return nil
}

117
magefiles/magefile.go Normal file
View file

@ -0,0 +1,117 @@
//go:build mage
package main
import (
"flow/services/internal"
"flow/services/internal/config"
"flow/services/internal/prepare"
"fmt"
"os"
"path/filepath"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
// Clean cleans the workspace.
func Clean(environment string) error {
buildDir := filepath.Join(internal.RootBuildDir, environment)
objects, err := os.ReadDir(buildDir)
if err != nil {
return err
}
for i := range objects {
name := objects[i].Name()
if name != ".gitkeep" {
if err := sh.Rm(buildDir + "/" + name); err != nil {
return err
}
}
}
return nil
}
// Deploy deploys the services to the Flow Platform.
func Deploy(environment, name string) error {
mg.Deps(
mg.F(Prepare, environment, name),
)
cfg, err := config.NewConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
os.Setenv("DOCKER_HOST", cfg.Docker.Host)
command := []string{
"docker",
"compose",
"--project-directory",
fmt.Sprintf("%s/%s/compose", internal.RootBuildDir, environment),
"up",
"-d",
"--build",
}
if name != "all" {
command = append(command, name)
}
return sh.RunV(command[0], command[1:]...)
}
// Prepare prepares the service's build directory.
func Prepare(environment, service string) error {
if err := prepare.Prepare(environment, service); err != nil {
return fmt.Errorf("error running preparations; %w", err)
}
return nil
}
// Shutdown shuts down the Flow Platform in a given environment.
func Shutdown(environment string) error {
cfg, err := config.NewConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
os.Setenv("DOCKER_HOST", cfg.Docker.Host)
command := []string{
"docker",
"compose",
"--project-directory",
fmt.Sprintf("%s/%s/compose", internal.RootBuildDir, environment),
"down",
}
return sh.RunV(command[0], command[1:]...)
}
// Stop stops a running service within the Flow Platform.
func Stop(environment, service string) error {
cfg, err := config.NewConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
os.Setenv("DOCKER_HOST", cfg.Docker.Host)
command := []string{
"docker",
"compose",
"--project-directory",
fmt.Sprintf("%s/%s/compose", internal.RootBuildDir, environment),
"stop",
service,
}
return sh.RunV(command[0], command[1:]...)
}

219
magefiles/prepare.go
View file

@ -1,219 +0,0 @@
//go:build mage
package main
import (
"fmt"
"io"
"io/fs"
"log"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/magefile/mage/mg"
)
// Prepare prepares the service's build directory.
func Prepare(environment, service string) error {
cfg, err := newConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %v", err)
}
if service == "all" {
objects, err := os.ReadDir(rootTemplatesDir)
if err != nil {
return fmt.Errorf("unable to read the templates directory; %w", err)
}
for _, o := range objects {
if !o.IsDir() {
continue
}
service := o.Name()
buildDir := filepath.Join(rootBuildDir, environment, service)
if _, err := os.Stat(buildDir); err != nil {
if err := os.MkdirAll(buildDir, 0o700); err != nil {
return fmt.Errorf("unable to make %s; %w", buildDir, err)
}
}
if service != "compose" {
mg.Deps(
mg.F(Download, environment, service),
)
log.Printf("Copying assets for %s.\n", service)
if err := copyAssets(environment, service); err != nil {
return fmt.Errorf("unable to copy the assets for %s; %w", service, err)
}
}
log.Printf("Rendering templates for %s.\n", service)
if err := render(cfg, environment, service); err != nil {
return fmt.Errorf("unable to render templates for %s; %w", service, err)
}
}
} else {
buildDir := filepath.Join(rootBuildDir, environment, service)
if _, err := os.Stat(buildDir); err != nil {
if err := os.MkdirAll(buildDir, 0o700); err != nil {
return fmt.Errorf("unable to make %s; %w", buildDir, err)
}
}
if service != "compose" {
mg.Deps(
mg.F(Download, environment, service),
mg.F(Prepare, environment, "compose"),
)
log.Printf("Copying assets for %s.\n", service)
if err := copyAssets(environment, service); err != nil {
return fmt.Errorf("unable to copy the assets for %s; %w", service, err)
}
}
if err := render(cfg, environment, service); err != nil {
return fmt.Errorf("an error occurred whilst rendering the templates; %w", err)
}
}
return nil
}
func render(cfg config, environment, component string) error {
buildDirName := filepath.Join(rootBuildDir, environment, component)
templateDirName := filepath.Join(rootTemplatesDir, component)
_, err := os.Stat(templateDirName)
if err != nil {
if os.IsNotExist(err) {
fmt.Printf("There's no template directory for %q.\n", component)
return nil
}
return err
}
files, err := os.ReadDir(templateDirName)
if err != nil {
return fmt.Errorf("unable to read files from %s; %w ", templateDirName, err)
}
funcMap := template.FuncMap{
"default": defaultValue,
}
for _, f := range files {
err := func() error {
templateFilename := f.Name()
if f.IsDir() || !strings.HasSuffix(templateFilename, templateExtension) {
return nil
}
outputFilename := strings.TrimSuffix(templateFilename, templateExtension)
outputPath := filepath.Join(buildDirName, outputFilename)
file, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("unable to create the file '%s'; %w", outputPath, err)
}
defer file.Close()
templatePath := filepath.Join(templateDirName, templateFilename)
tmpl, err := template.New(templateFilename).Funcs(funcMap).ParseFiles(templatePath)
if err != nil {
return fmt.Errorf("unable to create a new template value from '%s'; %w", templateFilename, err)
}
if err = tmpl.Execute(file, cfg); err != nil {
return fmt.Errorf("unable to render the template to '%s'; %w", outputPath, err)
}
return nil
}()
if err != nil {
return fmt.Errorf("an error occurred whilst rendering the templates for '%s'; %w", component, err)
}
}
return nil
}
func copyAssets(environment, service string) error {
assetsDirName := filepath.Join(rootAssetsDir, service)
if _, err := os.Stat(assetsDirName); err != nil {
if os.IsNotExist(err) {
fmt.Printf("There's no assets directory for %q.\n", service)
return nil
}
return err
}
buildDirName := filepath.Join(rootBuildDir, environment, service, "assets")
walkDirFunc := func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if path == assetsDirName {
return nil
}
buildAssetPath := filepath.Join(buildDirName, strings.TrimPrefix(path, assetsDirName))
if d.IsDir() {
if err := os.MkdirAll(buildAssetPath, 0o750); err != nil {
return fmt.Errorf("unable to make %s; %w", path, err)
}
return nil
}
source, err := os.Open(path)
if err != nil {
return err
}
defer source.Close()
dest, err := os.Create(buildAssetPath)
if err != nil {
return err
}
defer dest.Close()
if _, err := io.Copy(dest, source); err != nil {
return err
}
return nil
}
err := filepath.WalkDir(assetsDirName, walkDirFunc)
if err != nil {
return fmt.Errorf("error walking the path '%s'; %w", assetsDirName, err)
}
return nil
}
func defaultValue(value, defaultValue string) string {
if value == "" {
return defaultValue
}
return value
}

29
magefiles/shutdown.go
View file

@ -1,29 +0,0 @@
package main
import (
"fmt"
"os"
"github.com/magefile/mage/sh"
)
func Shutdown(environment string) error {
os.Setenv("MAGEFILE_VERBOSE", "true")
cfg, err := newConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
os.Setenv("DOCKER_HOST", cfg.Docker.Host)
command := []string{
"docker",
"compose",
"--project-directory",
fmt.Sprintf("%s/%s/compose", rootBuildDir, environment),
"down",
}
return sh.Run(command[0], command[1:]...)
}

30
magefiles/stop.go
View file

@ -1,30 +0,0 @@
package main
import (
"fmt"
"os"
"github.com/magefile/mage/sh"
)
func Stop(environment, service string) error {
os.Setenv("MAGEFILE_VERBOSE", "true")
cfg, err := newConfig(environment)
if err != nil {
return fmt.Errorf("unable to load the configuration; %w", err)
}
os.Setenv("DOCKER_HOST", cfg.Docker.Host)
command := []string{
"docker",
"compose",
"--project-directory",
fmt.Sprintf("%s/%s/compose", rootBuildDir, environment),
"stop",
service,
}
return sh.Run(command[0], command[1:]...)
}