mirror of
https://github.com/morbalint/kemkas-deployment.git
synced 2026-07-17 22:03:46 +00:00
init: basic k8s deployment on DO
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
*.pdf
|
||||
local/
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: SecretStore
|
||||
metadata:
|
||||
name: kemkas-secret-store
|
||||
namespace: kemkas
|
||||
spec:
|
||||
provider:
|
||||
onepassword:
|
||||
connectHost: https://1password-connect.dev.kemkas.hu
|
||||
vaults:
|
||||
kemkas: 1 # look in this vault first
|
||||
auth:
|
||||
secretRef:
|
||||
connectTokenSecretRef:
|
||||
name: onepassword-connect-token
|
||||
key: token
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: onepassword-connect-token
|
||||
namespace: kemkas
|
||||
type: Opaque
|
||||
stringData:
|
||||
token: <insert-token-from-1password>
|
||||
@@ -0,0 +1,14 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name 1password-connect.dev.kemkas.hu;
|
||||
|
||||
location / {
|
||||
proxy_pass_request_headers on;
|
||||
proxy_pass http://localhost:8080;
|
||||
}
|
||||
|
||||
error_page 500 502 503 504 /50x.html;
|
||||
location = /50x.html {
|
||||
root /usr/share/nginx/html;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
version: "3.4"
|
||||
|
||||
services:
|
||||
op-connect-api:
|
||||
image: 1password/connect-api:latest
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- "./1password-credentials.json:/home/opuser/.op/1password-credentials.json"
|
||||
- "data:/home/opuser/.op/data"
|
||||
op-connect-sync:
|
||||
image: 1password/connect-sync:latest
|
||||
ports:
|
||||
- "8081:8080"
|
||||
volumes:
|
||||
- "./1password-credentials.json:/home/opuser/.op/1password-credentials.json"
|
||||
- "data:/home/opuser/.op/data"
|
||||
|
||||
volumes:
|
||||
data:
|
||||
@@ -0,0 +1,79 @@
|
||||
# Kemkas k8s Deployment
|
||||
|
||||
## ToC
|
||||
- [Argo CD](#argo-cd)
|
||||
- [1password ESO connect server](#1password-connect-server)
|
||||
- [Kemkas](#kemkas)
|
||||
|
||||
## Argo CD
|
||||
|
||||
Goal: continous deployment automation with rollback capability
|
||||
|
||||
Files:
|
||||
- [argocd-ingress.yml](./argocd/argocd-ingress.yaml) for connecting the service with the load balancer and nginx reverse proxy
|
||||
- [argocd-values.yml](./argocd/argocd-values.yaml) config for the argocd helm chart to use SSL termination at NGINX.
|
||||
- [cert-manager-issuer.yaml](./argocd/cert-manager-issuer.yaml) SSL certificate config for the argocd.kemkas.hu domain
|
||||
|
||||
TODO:
|
||||
- [ ] gRPC setup for ArgoCD to use the CLI. Workaround: k8s port proxy to localhost.
|
||||
|
||||
Setup instructions
|
||||
|
||||
1. DO 1-click install argoCD into k8s cluster (TODO: replace with helm install command, I suspect behind the scenes the 1-click install does the same)
|
||||
1. DO 1-click install ingress-nginx into k8s cluster (TODO: replace with helm install command, I suspect behind the scenes the 1-click install does the same)
|
||||
1. DO 1-click install cert-manager into k8s cluster (TODO: replace with helm install command, I suspect behind the scenes the 1-click install does the same)
|
||||
1. Create DNS A record for argocd.kemkas.hu pointing to the Load Balancer IP address (Load balancer is a DO object created with the ingress-nginx helm chart install)
|
||||
1. helm update argocd with `argocd-values.yaml`
|
||||
1. Apply all files in `argocd` directory
|
||||
```
|
||||
kubectl apply -f cert-manager-issuer.yaml
|
||||
kubectl apply -f argocd-ingress.yaml
|
||||
```
|
||||
|
||||
## 1Password Connect Server
|
||||
|
||||
Goal: Store secrets in 1password instead of kubernetes, so we can push all kubernetes config while keeping the secrets safe in 1password
|
||||
|
||||
Files:
|
||||
- [1password-secret-store.yaml](./1password-connect-server/1password-secret-store.yaml): kuberentes ESO Secret Store to be referenced by external secrets
|
||||
- [connect-server-access-token-secret.yaml](./1password-connect-server/connect-server-access-token-secret.yaml): kubernetes secret file to connect with the 1password connect server without the actual secret token. the token can be generated on 1password website or CLI.
|
||||
- [docker-compose.yaml](./1password-connect-server/docker-compose.yaml): docker compose config to run the 1password connect server on a separate machine. Requires a `1password-credentials.json` file locally in the same directory, which can be obtained from the 1password website or CLI.
|
||||
- [connect-server.conf](./1password-connect-server/connect-server.conf) nginx config for the SSL terminating proxy for the 1password connect server (so that secrets don't travel unencrypted)
|
||||
|
||||
TODO:
|
||||
- [ ] explore running the 1password connect server on the same k8s cluster to simplify infrastructure
|
||||
- [ ] write script to automate the setup instraction (using `doctl` for full infra automation)
|
||||
|
||||
Setup:
|
||||
1. create DO droplet (with docker template)
|
||||
1. create DNS A record pointing to the droplet
|
||||
1. install nginx, certbot `sudo apt install nginx certbot python3-certbot-nginx`
|
||||
1. copy `docker-compose.yaml` and `1password-credentials.json` to the droplet and run `docker compose up -d`
|
||||
1. copy connect-server nginx config to `/etc/nginx/sites-available` and link it to sites enabled `ln -s /etc/nginx/sites-available/connect-server.conf /etc/nginx/sites-enabled/`
|
||||
1. test nginx config `nginx -t`
|
||||
1. reload nginx config `nginx -s reload`
|
||||
1. create SSL certificate with certbot `certbot --nginx -d 1password-connect.dev.kemkas.hu` (email address and accepting terms is required!)
|
||||
1. setup firewall to block TCP/8080 and TCP/8081 to avoid unencrypted access to the connect server (allow TCP/443 and TCP/22)
|
||||
1. helm install the external secret operator `helm repo add external-secrets https://charts.external-secrets.io` and `helm install external-secrets external-secrets/external-secrets -n external-secrets --create-namespace --set installCRDs=false` (from [https://external-secrets.io/](https://external-secrets.io/))
|
||||
1. create namespace for secret store `kubectl create ns kemkas`
|
||||
1. create connect server access token secret in k8s `kubectl apply -f ./1password-connect-server/connect-server-access-token-secret.yaml`
|
||||
1. create secret store `kubectl apply -f ./1password-connect-server/1password-secret-store.yaml`
|
||||
|
||||
## Kemkas
|
||||
|
||||
Goal: deploy an existing docker image into the k8s cluster with proper secrets and expose it on HTTPS with proper SSL cert
|
||||
|
||||
Files:
|
||||
- [kemkas-external-secrets.yaml](./kemkas/kemkas-external-secrets.yaml): syncs external secrets from the 1password secret store
|
||||
- [kemkas-deployment.yaml](./kemkas/kemkas-deployment.yaml): configures the kemkas server docker image to run on the cluster
|
||||
- [kemkas-service.yaml](./kemkas/kemkas-service.yaml): exposes the kemkas server deployment to the cluster
|
||||
- [kemkas-ssl-cert-issuer.yaml](./kemkas/kemkas-ssl-cert-issuer.yaml): config for issuing SSL certs for accessing the kemkas server on the internet
|
||||
- [kemkas-ingress.yaml](./kemkas/kemkas-ingress.yaml): exposes the kemkas server on the internet through the nginx ingress proxy
|
||||
|
||||
Setup:
|
||||
1. connect Digital Ocean Container Registry to the k8s cluster `doctl registry kubernetes-manifest | kubectl apply -f -`
|
||||
1. create external secrets for the upcoming kemkas deployment `kubectl apply -f ./kemkas/kemkas-external-secrets.yaml`
|
||||
1. create kemkas deployment `kubectl apply -f ./kemkas/kemkas-deployment.yaml`
|
||||
1. create service to export the kemkas deployment `kubectl apply -f ./kemkas/kemkas-service.yaml`
|
||||
1. create SSL cert issuer `kubectl apply -f ./kemkas/kemkas-ssl-cert-issuer.yaml`
|
||||
1. create ingress for the kemkas service `kubectl apply -f ./kemkas/kemkas-ingress.yaml`
|
||||
@@ -0,0 +1,26 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-argocd
|
||||
namespace: argocd
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
|
||||
nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
|
||||
cert-manager.io/issuer: letsencrypt-argocd
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: argocd-server
|
||||
port:
|
||||
name: http
|
||||
host: argocd.kemkas.hu
|
||||
tls:
|
||||
- hosts:
|
||||
- argocd.kemkas.hu
|
||||
secretName: letsencrypt-argocd
|
||||
@@ -0,0 +1,5 @@
|
||||
server:
|
||||
ingress:
|
||||
enabled: "true"
|
||||
extraArgs:
|
||||
- "--insecure"
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: letsencrypt-argocd
|
||||
namespace: argocd
|
||||
spec:
|
||||
# ACME issuer configuration
|
||||
# `email` - the email address to be associated with the ACME account (make sure it's a valid one)
|
||||
# `server` - the URL used to access the ACME server’s directory endpoint
|
||||
# `privateKeySecretRef` - Kubernetes Secret to store the automatically generated ACME account private key
|
||||
acme:
|
||||
email: developer@kemkas.hu
|
||||
server: https://acme-v02.api.letsencrypt.org/directory
|
||||
privateKeySecretRef:
|
||||
name: letsencrypt-argocd-private-key
|
||||
solvers:
|
||||
# Use the HTTP-01 challenge provider
|
||||
- http01:
|
||||
ingress:
|
||||
class: nginx
|
||||
@@ -0,0 +1,18 @@
|
||||
|
||||
controller:
|
||||
replicaCount: 1 #reduced to 1 to save costs
|
||||
# resources:
|
||||
# requests:
|
||||
# cpu: 0
|
||||
# memory: 90Mi
|
||||
service:
|
||||
type: LoadBalancer
|
||||
annotations:
|
||||
# Enable proxy protocol
|
||||
service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true"
|
||||
# Specify whether the DigitalOcean Load Balancer should pass encrypted data to backend droplets
|
||||
service.beta.kubernetes.io/do-loadbalancer-tls-passthrough: "true"
|
||||
|
||||
# Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
|
||||
config:
|
||||
use-proxy-protocol: "true"
|
||||
@@ -0,0 +1,77 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kemkas
|
||||
namespace: kemkas
|
||||
labels:
|
||||
app: kemkas
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kemkas
|
||||
template:
|
||||
metadata:
|
||||
name: kemkas
|
||||
labels:
|
||||
app: kemkas
|
||||
spec:
|
||||
containers:
|
||||
- name: kemkas
|
||||
image: registry.digitalocean.com/kemkas/kemkas:45914c78
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
- name: Email__DomainName
|
||||
value: mail.kemkas.hu
|
||||
- name: Email__ApiKey
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: email--api-key
|
||||
- name: ConnectionStrings__DefaultConnection
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: connection-strings--default-connection
|
||||
- name: Authentication__Google__ClientId
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: authentication--google--client-id
|
||||
- name: Authentication__Google__ClientSecret
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: authentication--google--client-secret
|
||||
- name: Authentication__Facebook__ClientId
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: authentication--facebook--client-id
|
||||
- name: Authentication__Facebook__ClientSecret
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: authentication--facebook--client-secret
|
||||
- name: Authentication__Discord__ClientId
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: authentication--discord--client-id
|
||||
- name: Authentication__Discord__ClientSecret
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: kemkas-envs
|
||||
key: authentication--discord--client-secret
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 100M
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 500M
|
||||
|
||||
restartPolicy: Always
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: kemkas-external-secrets
|
||||
namespace: kemkas
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
name: kemkas-secret-store
|
||||
kind: SecretStore
|
||||
target:
|
||||
name: kemkas-envs
|
||||
creationPolicy: Owner
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: kemkas-app
|
||||
@@ -0,0 +1,26 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-kemkas
|
||||
namespace: kemkas
|
||||
annotations:
|
||||
# nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
|
||||
# nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
|
||||
cert-manager.io/issuer: letsencrypt-kemkas
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: kemkas-service
|
||||
port:
|
||||
number: 8080
|
||||
host: dev.kemkas.hu
|
||||
tls:
|
||||
- hosts:
|
||||
- dev.kemkas.hu
|
||||
secretName: letsencrypt-kemkas
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kemkas-service
|
||||
namespace: kemkas
|
||||
labels:
|
||||
app: kemkas
|
||||
spec:
|
||||
selector:
|
||||
app: kemkas
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: letsencrypt-kemkas
|
||||
namespace: kemkas
|
||||
spec:
|
||||
# ACME issuer configuration
|
||||
# `email` - the email address to be associated with the ACME account (make sure it's a valid one)
|
||||
# `server` - the URL used to access the ACME server’s directory endpoint
|
||||
# `privateKeySecretRef` - Kubernetes Secret to store the automatically generated ACME account private key
|
||||
acme:
|
||||
email: developer@kemkas.hu
|
||||
server: https://acme-v02.api.letsencrypt.org/directory
|
||||
privateKeySecretRef:
|
||||
name: letsencrypt-kemkas-private-key
|
||||
solvers:
|
||||
# Use the HTTP-01 challenge provider
|
||||
- http01:
|
||||
ingress:
|
||||
class: nginx
|
||||
Reference in New Issue
Block a user