From fda0d6a6823f63067cf7f604446cff10fcef6c00 Mon Sep 17 00:00:00 2001 From: Dan Anglin Date: Sat, 10 Jul 2021 12:41:15 +0100 Subject: [PATCH] feat: add traefik in the new containers stack This commit adds the new containers stack which now builds the traefik docker image and creates the traefik docker container. Pulumi needed to be downgraded to version 3.2.1 because later versions panic when building the docker image. --- go.mod | 2 +- go.sum | 4 +- internal/config/config.go | 24 +++- internal/config/config_test.go | 9 ++ internal/config/testdata/config-invalid.json | 9 ++ internal/config/testdata/config-valid.json | 9 ++ internal/docker/container.go | 19 +-- internal/docker/image.go | 36 +++-- internal/stacks/containers.go | 129 ++++++++++++++++++ internal/stacks/docker_network.go | 2 +- internal/stacks/manage.go | 126 ++++++++++------- internal/stacks/templates.go | 24 ++++ .../stacks/templates/traefik/Dockerfile.tmpl | 7 + .../templates/traefik/traefik.yaml.tmpl | 27 ++++ 14 files changed, 351 insertions(+), 76 deletions(-) create mode 100644 internal/stacks/containers.go create mode 100644 internal/stacks/templates.go create mode 100644 internal/stacks/templates/traefik/Dockerfile.tmpl create mode 100644 internal/stacks/templates/traefik/traefik.yaml.tmpl diff --git a/go.mod b/go.mod index 8fefd89..b003689 100644 --- a/go.mod +++ b/go.mod @@ -5,5 +5,5 @@ go 1.16 require ( github.com/leaanthony/clir v1.0.4 github.com/pulumi/pulumi-docker/sdk/v3 v3.0.0 - github.com/pulumi/pulumi/sdk/v3 v3.6.0 + github.com/pulumi/pulumi/sdk/v3 v3.2.1 ) diff --git a/go.sum b/go.sum index 79eea32..47edaa4 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/pulumi/pulumi-docker/sdk/v3 v3.0.0 h1:torA+0p0G14PaEmK9RO2/bvXsWeuko5Y+en+dJsUNPU= github.com/pulumi/pulumi-docker/sdk/v3 v3.0.0/go.mod h1:KusFPDVt8YTZj58vpa7gJyQyXoPkrHOKyw5k06bT340= github.com/pulumi/pulumi/sdk/v3 v3.0.0/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds= -github.com/pulumi/pulumi/sdk/v3 v3.6.0 h1:ZuOGcDwZiuiNyfXH8o2ooy2qBvpGtqft47+9mDOHS70= -github.com/pulumi/pulumi/sdk/v3 v3.6.0/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds= +github.com/pulumi/pulumi/sdk/v3 v3.2.1 h1:gHeuYmOR/GSDYhGJfdTbY8SVEHyIts4pq6wiuIXZyfc= +github.com/pulumi/pulumi/sdk/v3 v3.2.1/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0= diff --git a/internal/config/config.go b/internal/config/config.go index f25790d..dc80893 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,10 +7,13 @@ import ( ) type Config struct { - ProjectName string `json:"project"` - Docker DockerConfig `json:"docker"` + ProjectName string `json:"project"` + Docker DockerConfig `json:"docker"` + Services ServicesConfig `json:"services"` } +// DockerConfig contains the configuration for +// docker specific components. type DockerConfig struct { Network DockerNetworkConfig `json:"network"` } @@ -23,6 +26,23 @@ type DockerNetworkConfig struct { Driver string `json:"driver"` } +// Services contains a list of +// services and their configuration +type ServicesConfig struct { + Traefik TraefikConfig `json:"traefik"` +} + +// TraefikConfig contains configuration for the Traefik container. +type TraefikConfig struct { + CheckNewVersion bool `json:"checkNewVersion"` + SendAnonymousUsage bool `json:"sendAnonymousUsage"` + Version string `json:"version"` + ContainerIp string `json:"containerIp"` + LogLevel string `json:"logLevel"` +} + +// NewConfig creates a new Config value from a given +// JSON file. func NewConfig(file string) (Config, error) { var c Config var err error diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c2b87ff..d83bc8f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,15 @@ func TestValidConfig(t *testing.T) { Driver: "default", }, }, + Services: ServicesConfig{ + Traefik: TraefikConfig{ + CheckNewVersion: true, + SendAnonymousUsage: false, + Version: "v2.4.9", + ContainerIp: "172.17.1.2", + LogLevel: "info", + }, + }, }, } diff --git a/internal/config/testdata/config-invalid.json b/internal/config/testdata/config-invalid.json index f225726..1a5ab27 100644 --- a/internal/config/testdata/config-invalid.json +++ b/internal/config/testdata/config-invalid.json @@ -7,4 +7,13 @@ "driver": "default" } } + "services": { + "traefik": { + "checkNewVersion": true, + "sendAnonymousUsage": false, + "version": "v2.4.9", + "containerIp": "172.17.1.2", + "logLevel": "info" + } + } } diff --git a/internal/config/testdata/config-valid.json b/internal/config/testdata/config-valid.json index 3d1f103..a353b0a 100644 --- a/internal/config/testdata/config-valid.json +++ b/internal/config/testdata/config-valid.json @@ -6,5 +6,14 @@ "name": "forge-platform-test-netwwork", "driver": "default" } + }, + "services": { + "traefik": { + "checkNewVersion": true, + "sendAnonymousUsage": false, + "version": "v2.4.9", + "containerIp": "172.17.1.2", + "logLevel": "info" + } } } diff --git a/internal/docker/container.go b/internal/docker/container.go index bd21bf9..a301988 100644 --- a/internal/docker/container.go +++ b/internal/docker/container.go @@ -7,26 +7,27 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -// DockerContainerConfig is the configuration +// DockerContainerInput is the configuration // used to create the Gitea docker container. -type DockerContainerConfig struct { +type DockerContainerInput struct { Image pulumi.StringInput Ipv4Address pulumi.StringInput EnvVars pulumi.StringArray Name pulumi.StringInput Network pulumi.StringInput - Volumes []DockerVolumeConfig + Volumes []DockerVolumeMount + UniqueLabel string } -// VolumeConfig is the configuration +// DockerVolumeMount is the configuration // used for mounting a host directory // to a directory inside a container. -type DockerVolumeConfig struct { +type DockerVolumeMount struct { HostPath pulumi.StringInput MountPath pulumi.StringInput } -func CreateDockerContainer(ctx *pulumi.Context, c DockerContainerConfig) error { +func CreateDockerContainer(ctx *pulumi.Context, c DockerContainerInput) error { // all containers will mount the host's timezone and localtime files // to ensure the correct time is synced. volumes := []docker.ContainerVolumeInput{ @@ -42,7 +43,7 @@ func CreateDockerContainer(ctx *pulumi.Context, c DockerContainerConfig) error { }, } - // create additional (optional) container volumes. + // optionally create additional container volumes. for _, v := range c.Volumes { vArg := docker.ContainerVolumeArgs{ ContainerPath: v.MountPath, @@ -66,9 +67,9 @@ func CreateDockerContainer(ctx *pulumi.Context, c DockerContainerConfig) error { Volumes: docker.ContainerVolumeArray(volumes), } - _, err := docker.NewContainer(ctx, "gitea_container", &args) + _, err := docker.NewContainer(ctx, fmt.Sprintf("%s_container", c.UniqueLabel), &args) if err != nil { - return fmt.Errorf("unable to create the Gitea docker container, %w", err) + return fmt.Errorf("unable to create the Docker container for %s...\n%w", c.UniqueLabel, err) } return nil diff --git a/internal/docker/image.go b/internal/docker/image.go index cb94cbc..f7a9b85 100644 --- a/internal/docker/image.go +++ b/internal/docker/image.go @@ -7,28 +7,44 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -// DockerImageConfig is the configuration +// DockerImageInput is the configuration // used to create the local Gitea docker // image. -type DockerImageConfig struct { - ImageName pulumi.StringInput +type DockerImageInput struct { BuildContext pulumi.StringInput - Version pulumi.StringInput + Dockerfile pulumi.StringInput + ImageName pulumi.StringInput + ImageTag pulumi.StringInput + UniqueLabel string +} + +// DockerImageOutput contains the details +// of the generated Docker image. +type DockerImageOutput struct { + ImageName pulumi.StringOutput } // CreateDockerImage creates a local Docker image. -func CreateDockerImage(ctx *pulumi.Context, c DockerImageConfig) error { +func CreateDockerImage(ctx *pulumi.Context, c DockerImageInput) (DockerImageOutput, error) { + var output DockerImageOutput + args := docker.ImageArgs{ - ImageName: c.ImageName, + ImageName: pulumi.Sprintf("%s:%s", c.ImageName, c.ImageTag), SkipPush: pulumi.Bool(true), Build: docker.DockerBuildArgs{ - Context: c.BuildContext, + Context: c.BuildContext, + Dockerfile: c.Dockerfile, }, } - _, err := docker.NewImage(ctx, "gitea_image", &args) + i, err := docker.NewImage(ctx, fmt.Sprintf("%s_image", c.UniqueLabel), &args) if err != nil { - return fmt.Errorf("unable to create the Gitea docker image, %w", err) + return output, fmt.Errorf("unable to create the Docker image for %s...\n%w", c.UniqueLabel, err) } - return nil + + output = DockerImageOutput{ + ImageName: i.BaseImageName, + } + + return output, nil } diff --git a/internal/stacks/containers.go b/internal/stacks/containers.go new file mode 100644 index 0000000..a733d24 --- /dev/null +++ b/internal/stacks/containers.go @@ -0,0 +1,129 @@ +package stacks + +import ( + "context" + _ "embed" + "fmt" + "os" + "path/filepath" + + "github.com/pulumi/pulumi/sdk/v3/go/auto" + "github.com/pulumi/pulumi/sdk/v3/go/auto/optdestroy" + "github.com/pulumi/pulumi/sdk/v3/go/auto/optpreview" + "github.com/pulumi/pulumi/sdk/v3/go/auto/optup" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "gitlab.com/dananglin/helix/internal/config" + "gitlab.com/dananglin/helix/internal/docker" +) + +// ContainerStack is a stack for managing the containers +// for the forge platform. +type ContainerStack struct { + Name string + Stack auto.Stack +} + +//go:embed templates/traefik/Dockerfile.tmpl +var templateTraefikDockerfile string + +//go:embed templates/traefik/traefik.yaml.tmpl +var templateTraefikStaticConfig string + +// newContainerStack creates the ContainerStack value. +func newContainerStack(ctx context.Context, project, stack, dockerNetwork string, conf config.TraefikConfig) (*ContainerStack, error) { + deployFunc := deployContainerStack(project, dockerNetwork, conf) + + s, err := createOrSelectStack(ctx, project, stack, deployFunc) + if err != nil { + return nil, fmt.Errorf("unable to initialise the '%s' stack...\n%w", stack, err) + } + + c := ContainerStack{ + Name: stack, + Stack: s, + } + + return &c, nil +} + +// Preview the proposed changes to the container stack. +func (c *ContainerStack) Preview(ctx context.Context) error { + streamer := optpreview.ProgressStreams(os.Stdout) + _, err := c.Stack.Preview(ctx, streamer) + if err != nil { + return fmt.Errorf("unable to preview the '%s' stack...\n%w", c.Name, err) + } + return nil +} + +// Update the container stack. +func (c *ContainerStack) Update(ctx context.Context) error { + streamer := optup.ProgressStreams(os.Stdout) + _, err := c.Stack.Up(ctx, streamer) + if err != nil { + return fmt.Errorf("unable to update '%s' stack...\n%w", c.Name, err) + } + return nil +} + +// Destroy the container stack. +func (c *ContainerStack) Destroy(ctx context.Context) error { + streamer := optdestroy.ProgressStreams(os.Stdout) + _, err := c.Stack.Destroy(ctx, streamer) + if err != nil { + return fmt.Errorf("unable to destroy '%s' stack...\n%w", c.Name, err) + } + return nil +} + +// deployContainerStack returns a Pulumi run function +// that is used to deploy the container stack. +func deployContainerStack(project, dockerNetwork string, t config.TraefikConfig) pulumi.RunFunc { + return func(ctx *pulumi.Context) error { + base_cache, err := os.UserCacheDir() + if err != nil { + return fmt.Errorf("unable to get the base cache directory...\n%w", err) + } + + traefikContextDir := filepath.Join(base_cache, "helix", project, "traefik") + + if err := os.MkdirAll(traefikContextDir, 0700); err != nil { + return fmt.Errorf("unable to make the cache directory for traefik...\n%w", err) + } + + if err := generateFile(t, templateTraefikDockerfile, "traefikDocker", filepath.Join(traefikContextDir, "Dockerfile")); err != nil { + return fmt.Errorf("unable to generate the Traefik Dockerfile from template...\n%w", err) + } + + if err := generateFile(t, templateTraefikStaticConfig, "traefikStaticConf", filepath.Join(traefikContextDir, "traefik.yml")); err != nil { + return fmt.Errorf("unable to generate the Traefik static configuration from template...\n%w", err) + } + + c := docker.DockerImageInput{ + BuildContext: pulumi.String(traefikContextDir), + Dockerfile: pulumi.String(filepath.Join(traefikContextDir, "Dockerfile")), + ImageName: pulumi.String("helix-traefik"), + ImageTag: pulumi.String(t.Version), + UniqueLabel: "traefik-image", + } + + traefikImage, err := docker.CreateDockerImage(ctx, c) + if err != nil { + return err + } + + traefikContainerInput := docker.DockerContainerInput{ + Image: traefikImage.ImageName, + Ipv4Address: pulumi.String(t.ContainerIp), + Name: pulumi.String("helix-traefik"), + Network: pulumi.String(dockerNetwork), + UniqueLabel: "traefik-container", + } + + if err = docker.CreateDockerContainer(ctx, traefikContainerInput); err != nil { + return err + } + + return nil + } +} diff --git a/internal/stacks/docker_network.go b/internal/stacks/docker_network.go index d7f073c..dbb7d4d 100644 --- a/internal/stacks/docker_network.go +++ b/internal/stacks/docker_network.go @@ -68,7 +68,7 @@ func (d *DockerNetworkStack) Destroy(ctx context.Context) error { return nil } -// DeployDockerNetworkStack returns a Pulumi run function +// deployDockerNetworkStack returns a Pulumi run function // that deploys the Docker network stack. func deployDockerNetworkStack(name, subnet, driver string) pulumi.RunFunc { return func(ctx *pulumi.Context) error { diff --git a/internal/stacks/manage.go b/internal/stacks/manage.go index b014af1..ee2ddfa 100644 --- a/internal/stacks/manage.go +++ b/internal/stacks/manage.go @@ -16,6 +16,7 @@ import ( const ( dockerNetworkStackName string = "docker_network" + containerStackName string = "containers" ) type Previewer interface { @@ -30,6 +31,72 @@ type Destroyer interface { Destroy(ctx context.Context) error } +func NewPreviewer(ctx context.Context, stack string, c config.Config) (Previewer, error) { + var p Previewer + var err error + + switch stack { + case dockerNetworkStackName: + p, err = newDockerNetworkStack(ctx, c.ProjectName, stack, c.Docker.Network) + if err != nil { + return nil, fmt.Errorf("unable to initialise '%s' stack...\n%w", stack, err) + } + case containerStackName: + p, err = newContainerStack(ctx, c.ProjectName, stack, c.Docker.Network.Name, c.Services.Traefik) + if err != nil { + return nil, fmt.Errorf("unable to initialise '%s' stack...\n%w", stack, err) + } + default: + return nil, fmt.Errorf("unknown stack name '%s'", stack) + } + + return p, nil +} + +func NewUpdater(ctx context.Context, stack string, c config.Config) (Updater, error) { + var u Updater + var err error + + switch stack { + case dockerNetworkStackName: + u, err = newDockerNetworkStack(ctx, c.ProjectName, stack, c.Docker.Network) + if err != nil { + return nil, fmt.Errorf("unable to initialise '%s' stack...\n%w", stack, err) + } + case containerStackName: + u, err = newContainerStack(ctx, c.ProjectName, stack, c.Docker.Network.Name, c.Services.Traefik) + if err != nil { + return nil, fmt.Errorf("unable to initialise '%s' stack...\n%w", stack, err) + } + default: + return nil, fmt.Errorf("unknown stack name '%s'", stack) + } + + return u, nil +} + +func NewDestroyer(ctx context.Context, stack string, c config.Config) (Destroyer, error) { + var d Destroyer + var err error + + switch stack { + case dockerNetworkStackName: + d, err = newDockerNetworkStack(ctx, c.ProjectName, stack, c.Docker.Network) + if err != nil { + return nil, fmt.Errorf("unable to initialise '%s' stack...\n%w", stack, err) + } + case containerStackName: + d, err = newContainerStack(ctx, c.ProjectName, stack, c.Docker.Network.Name, c.Services.Traefik) + if err != nil { + return nil, fmt.Errorf("unable to initialise '%s' stack...\n%w", stack, err) + } + default: + return nil, fmt.Errorf("unknown stack name '%s'", stack) + } + + return d, nil +} + func createOrSelectStack(ctx context.Context, projectName, stackName string, deployFunc pulumi.RunFunc) (auto.Stack, error) { var s auto.Stack @@ -44,6 +111,14 @@ func createOrSelectStack(ctx context.Context, projectName, stackName string, dep return s, fmt.Errorf("unable to create/select the stack...\n%w", err) } + w := s.Workspace() + + fmt.Println("Installing the Docker plugin") + + if err = w.InstallPlugin(ctx, "docker", "v2.10.0"); err != nil { + return s, fmt.Errorf("unable to install the docker plugin...\n%w", err) + } + fmt.Printf("Refreshing stack (%s)...\n", stackName) _, err = s.Refresh(ctx) if err != nil { @@ -84,54 +159,3 @@ func workspaceOptions(projectName, stackName string) ([]auto.LocalWorkspaceOptio return opts, nil } - -func NewPreviewer(ctx context.Context, stack string, c config.Config) (Previewer, error) { - var p Previewer - var err error - - switch stack { - case dockerNetworkStackName: - p, err = newDockerNetworkStack(ctx, c.ProjectName, stack, c.Docker.Network) - if err != nil { - return nil, fmt.Errorf("unable to initialise '%s' stack...\n%v", stack, err) - } - default: - return nil, fmt.Errorf("unknown stack name '%s'", stack) - } - - return p, nil -} - -func NewUpdater(ctx context.Context, stack string, c config.Config) (Updater, error) { - var u Updater - var err error - - switch stack { - case dockerNetworkStackName: - u, err = newDockerNetworkStack(ctx, c.ProjectName, stack, c.Docker.Network) - if err != nil { - return nil, fmt.Errorf("unable to initialise '%s' stack...\n%v", stack, err) - } - default: - return nil, fmt.Errorf("unknown stack name '%s'", stack) - } - - return u, nil -} - -func NewDestroyer(ctx context.Context, stack string, c config.Config) (Destroyer, error) { - var d Destroyer - var err error - - switch stack { - case dockerNetworkStackName: - d, err = newDockerNetworkStack(ctx, c.ProjectName, stack, c.Docker.Network) - if err != nil { - return nil, fmt.Errorf("unable to initialise '%s' stack...\n%v", stack, err) - } - default: - return nil, fmt.Errorf("unknown stack name '%s'", stack) - } - - return d, nil -} diff --git a/internal/stacks/templates.go b/internal/stacks/templates.go new file mode 100644 index 0000000..7b0a97a --- /dev/null +++ b/internal/stacks/templates.go @@ -0,0 +1,24 @@ +package stacks + +import ( + "fmt" + "os" + "text/template" +) + +// generateFile renders a given template to a given filepath. +func generateFile(data interface{}, templateString, templateName, path string) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("unable to create the file '%s'...\n%v", path, err) + } + defer file.Close() + + tmpl := template.Must(template.New(templateName).Parse(templateString)) + + if err = tmpl.Execute(file, data); err != nil { + return fmt.Errorf("unable to execute the template at '%s'...\n%v", path, err) + } + + return nil +} diff --git a/internal/stacks/templates/traefik/Dockerfile.tmpl b/internal/stacks/templates/traefik/Dockerfile.tmpl new file mode 100644 index 0000000..798dc1f --- /dev/null +++ b/internal/stacks/templates/traefik/Dockerfile.tmpl @@ -0,0 +1,7 @@ +FROM traefik:{{ .Version }} + +ADD traefik.yml /helix/traefik/ + +EXPOSE 22 80 443 + +CMD ["--configfile=/helix/traefik/traefik.yml"] diff --git a/internal/stacks/templates/traefik/traefik.yaml.tmpl b/internal/stacks/templates/traefik/traefik.yaml.tmpl new file mode 100644 index 0000000..cf0acf6 --- /dev/null +++ b/internal/stacks/templates/traefik/traefik.yaml.tmpl @@ -0,0 +1,27 @@ +--- +global: + checkNewVersion: {{ .CheckNewVersion }} + sendAnonymousUsage: {{ .SendAnonymousUsage }} +api: + insecure: false + dashboard: true + debug: false +entryPoints: + http: + address: "{{ .ContainerIp }}:80" + http: + redirections: + entryPoint: + to: "https" + scheme: "https" + permanent: true + https: + address: "{{ .ContainerIp }}:443" + ssh: + address: "{{ .ContainerIp }}:22" +providers: + file: + watch: true + directory: "/helix/traefik/config/dynamic" +log: + level: "{{ .LogLevel }}"