diff --git a/.github/workflows/release-nixos.yml b/.github/workflows/release-nixos.yml index a622173..decad5a 100644 --- a/.github/workflows/release-nixos.yml +++ b/.github/workflows/release-nixos.yml @@ -20,23 +20,9 @@ jobs: runs-on: ubuntu-latest name: Release nix image steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: cachix/install-nix-action@v25 - with: - extra_nix_config: | - experimental-features = nix-command flakes - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Restore and cache Nix store - uses: nix-community/cache-nix-action@v5 - with: - primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }} - restore-prefixes-first-match: nix-${{ runner.os }}- - gc-max-store-size-linux: 1073741824 + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - name: Build production image id: build-stable diff --git a/Makefile b/Makefile index 54bb42e..2fc929f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ MAKEFLAGS += --no-builtin-rules --no-builtin-variables TF_CMD:=apply -auto-approve VARIANT=builder SYSTEM?=aarch64-linux +TESTING_X86_URL=https://github.com/loic-roux-404/k3s-paas/releases/download/nixos-testing/nixos.qcow2 #### Nix @@ -28,6 +29,11 @@ nixos-local: bootstrap build build: @nix build .#nixosConfigurations.initial.config.formats.qcow --system $(SYSTEM) +pull-testing-x86: + @rm -rf result && mkdir result + @wget -q --show-progress -O result/nixos.qcow2 $(TESTING_X86_URL) + @qemu-img resize result/nixos.qcow2 16G + TERRAGRUNT_FILES:=$(shell find terragrunt -type d -name '.*' -prune -o -name 'terragrunt.hcl' -exec dirname {} \;) $(TERRAGRUNT_FILES): diff --git a/nixos-options/default.nix b/nixos-options/default.nix index ebefe35..8fca24b 100644 --- a/nixos-options/default.nix +++ b/nixos-options/default.nix @@ -1,4 +1,4 @@ -{ lib, ... }: +{ lib, pkgs, config, ... }: { options.k3s-paas = { @@ -62,11 +62,82 @@ default = ""; }; + k3s.podCIDR = lib.mkOption { + type = lib.types.str; + description = "Pod CIDR"; + default = "10.100.0.0/16"; + }; + + k3s.serviceCIDR = lib.mkOption { + type = lib.types.str; + description = "Pod CIDR"; + default = "10.110.0.0/16"; + }; + + k3s.clusterDns = lib.mkOption { + type = lib.types.str; + description = "Cluster DNS"; + default = "10.110.0.10"; + }; + + k3s.serviceIp = lib.mkOption { + type = lib.types.str; + description = "Service IP"; + default = "10.110.0.1"; + }; + + k3s.serviceHost = lib.mkOption { + type = lib.types.str; + description = "Service host"; + default = ""; + }; + + k3s.servicePort = lib.mkOption { + type = lib.types.int; + description = "Service port"; + default = 6443; + }; + + cilium.version = lib.mkOption { + type = lib.types.str; + description = "Cilium version"; + default = "1.16.1"; + }; + dex.dexClientId = lib.mkOption { type = lib.types.str; description = "Client ID for Dex"; default = "dex-k3s-paas"; }; + cert-manager.version = lib.mkOption { + type = lib.types.str; + description = "Cert Manager version"; + default = "1.15.2"; + }; + + defaultK3sConfigPath = lib.mkOption { + type = lib.types.str; + description = "Default config yaml"; + default = ""; + }; + + }; + + config = with config.k3s-paas; { + k3s-paas.defaultK3sConfigPath = pkgs.writeText '' + cluster-cidr: ${k3s.podCIDR} + service-cidr: ${k3s.serviceCIDR} + cluster-dns: ${k3s.clusterDns} + tls-san: + - localhost + - ${k3s.serviceIp} + - ${config.networking.hostName} + kube-apiserver-arg=authorization-mode: Node,RBAC + kube-apiserver-arg=oidc-issuer-url: https://dex.${dns.name} + kube-apiserver-arg=oidc-client-id: ${dex.dexClientId} + kube-apiserver-arg=oidc-username-claim: email + kube-apiserver-arg=oidc-groups-claim: groups + ''; }; } diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 848851b..e625fed 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -82,7 +82,7 @@ in { config.services.tailscale.enable ) { serviceConfig = { - RemainAfterExit = true; # Ensures it's remains active after running. + RemainAfterExit = true; }; }; @@ -105,7 +105,6 @@ in { services.k3s = { enable = lib.mkDefault false; role = "server"; - package = k3sPkg; extraFlags = lib.strings.concatStringsSep " " ( map (service: "--disable=${service}") k3s.disableServices ++ k3s.serverExtraArgs @@ -116,25 +115,93 @@ in { "--egress-selector-mode=disabled" ] ); - # manifests = { - # certManager = { - # name = "cert-manager"; - # namespace = certManagerNamespace; - # createNamespace = true; - # repository = "https://charts.jetstack.io"; - # chart = "cert-manager"; - # version = "1.15.2"; - # waitForJobs = true; - # atomic = true; - # timeout = 120; - - # values = '' - # crds: - # enabled = true - # ''; - # }; - # }; + configPath = k3s.defaultK3sConfigPath; + manifests = { + certManager = { + name = "cert-manager"; + namespace = certManagerNamespace; + createNamespace = true; + repository = "https://charts.jetstack.io"; + chart = "cert-manager"; + version = cert-manager.version; + waitForJobs = true; + atomic = true; + timeout = 120; + + values = '' + crds: + enabled = true + ''; + }; + cilium = { + name = "cilium"; + namespace = var.cilium_namespace; + repository = "https://helm.cilium.io"; + chart = "cilium"; + atomic = true; + version = cilium.version; + values = '' + l2announcements: + enabled: true + + kubeProxyReplacement: true + + bpf: + masquerade: true + lbExternalClusterIP: false + + gatewayAPI: + enabled: false + + routingMode: tunnel + + tunnelProtocol: vxlan + + ingressController: + enabled: true + default: true + loadbalancerMode: dedicated + service: + name: cilium-ingress-external + labels: + k3s-paas/internal: "true" + + prometheus: + enabled: true + serviceMonitor: + enabled: true + + operator: + replicas: 1 + prometheus: + enabled: true + + hubble: + relay: + enabled: true + metrics: + enabled: + - dns + - drop + - tcp + - flow + - port-distribution + - icmp + - httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction + enableOpenMetrics: true + + ipam: + operator: + clusterPoolIPv4PodCIDRList: + - "${k3s.podCIDR}" + '' + lib.mkIf (k3s.serviceHost != "") '' + k8sServiceHost: "${k3s.serviceHost}" + k8sServicePort: "${k3s.servicePort}" + ''; + }; + }; }; + services.fail2ban.enable = true; security.pki.certificateFiles = certs; diff --git a/nixos/deploy.nix b/nixos/deploy.nix index c60603d..385cb97 100644 --- a/nixos/deploy.nix +++ b/nixos/deploy.nix @@ -44,13 +44,13 @@ with config.k3s-paas; node-name: "${config.networking.hostName}" cluster-domain: ${config.sops.placeholder.paasDomain} node-external-ip: "${config.sops.placeholder.nodeIp}" - cluster-cidr: 10.100.0.0/16 - service-cidr: 10.110.0.0/16 - cluster-dns: 10.110.0.10 + cluster-cidr: ${k3s.podCIDR} + service-cidr: ${k3s.serviceCIDR} + cluster-dns: ${k3s.clusterDns} vpn-auth: "name=tailscale,joinKey=${config.sops.placeholder.tailscaleNodeKey}" tls-san: - localhost - - 10.43.0.1 + - ${k3s.serviceIp} - ${config.networking.hostName} - "${config.sops.placeholder.tailscaleDomain}" - "${config.sops.placeholder.nodeIp}" diff --git a/nixos/qcow-compressed.nix b/nixos/qcow-compressed.nix index e1d2c8e..9034156 100644 --- a/nixos/qcow-compressed.nix +++ b/nixos/qcow-compressed.nix @@ -4,7 +4,7 @@ ]; system.build.qcow = lib.mkForce (import "${toString modulesPath}/../lib/make-disk-image.nix" { inherit lib config pkgs; - diskSize = "auto"; + diskSize = 8192; format = "qcow2-compressed"; partitionTableType = "hybrid"; }); diff --git a/tf-modules-ai/librechat/main.tf b/tf-modules-ai/librechat/main.tf new file mode 100644 index 0000000..c455377 --- /dev/null +++ b/tf-modules-ai/librechat/main.tf @@ -0,0 +1,70 @@ + + +resource "random_string" "creds_key" { + length = 64 + special = false +} + +resource "random_string" "creds_iv" { + length = 32 + special = false +} + +resource "random_string" "jwt_secret" { + length = 64 + special = false +} + +resource "random_string" "jwt_refresh_secret" { + length = 64 + special = false +} + +resource "kubernetes_secret" "librechat" { + metadata { + name = "librechat" + namespace = "default" + } + + data = { + CREDS_KEY = "${random_string.creds_key.result}" + CREDS_IV = "${random_string.creds_iv.result}" + MONGO_URI = "${helm_release.mongodb.output.mongodb_uri}" + JWT_SECRET = "${random_string.jwt_secret.result}" + JWT_REFRESH_SECRET = "${random_string.jwt_refresh_secret.result}" + } +} + +resource "helm_release" "librechat" { + name = "librechat" + chart = "" # Path to the LibreChat chart. + namespace = "default" + + + values = [ + yamlencode({ + config = { + env_secrets = { + secret_ref = kubernetes_secret.librechat.metadata[0].name + } + } + + env = { + ALLOW_EMAIL_LOGIN = true + ALLOW_REGISTRATION = true + ALLOW_SOCIAL_LOGIN = false + ALLOW_SOCIAL_REGISTRATION = false + CUSTOM_FOOTER = "Orga-404 librechat" + DEBUG_CONSOLE = true + DEBUG_LOGGING = true + DEBUG_OPENAI = true + DEBUG_PLUGINS = true + DOMAIN_CLIENT = "" + DOMAIN_SERVER = "" + ENDPOINTS = "openAI,azureOpenAI,bingAI,chatGPTBrowser,google,gptPlugins,anthropic" + MONGO_URI = "mongodb://${var.mongo_user}:${var.mongo_password}@${var.mongo_host}:${var.mongo_port}/${var.mongo_database}" + } + }) + ] +} + diff --git a/tf-modules-ai/librechat/terraform.tf b/tf-modules-ai/librechat/terraform.tf new file mode 100644 index 0000000..62026e5 --- /dev/null +++ b/tf-modules-ai/librechat/terraform.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + mongodb = { + source = "01Joseph-Hwang10/mongodb" + } + } +} diff --git a/tf-modules-ai/librechat/variables.tf b/tf-modules-ai/librechat/variables.tf new file mode 100644 index 0000000..96f6807 --- /dev/null +++ b/tf-modules-ai/librechat/variables.tf @@ -0,0 +1,20 @@ +variable "mongo_database" { + description = "The name of the MongoDB database to use." + default = "librechat" +} + +variable "mongo_host" { + description = "The hostname of the MongoDB server." +} + +variable "mongo_password" { + description = "The password to use to connect to the MongoDB server." +} + +variable "mongo_user" { + description = "The username to use to connect to the MongoDB server." +} + +variable "mongo_port" { + default = 27017 +} diff --git a/tf-modules-cloud/libvirt/main.tf b/tf-modules-cloud/libvirt/main.tf index cb31752..74dcfcb 100644 --- a/tf-modules-cloud/libvirt/main.tf +++ b/tf-modules-cloud/libvirt/main.tf @@ -23,6 +23,7 @@ resource "libvirt_volume" "nixos_worker" { name = "nixos-worker.qcow2" base_volume_id = libvirt_volume.nixos.id pool = libvirt_pool.volumetmp.name + size = 16 * 1024 * 1024 * 1024 } resource "libvirt_domain" "machine" { diff --git a/tf-modules-cloud/libvirt/variables.tf b/tf-modules-cloud/libvirt/variables.tf index 3324396..093e999 100644 --- a/tf-modules-cloud/libvirt/variables.tf +++ b/tf-modules-cloud/libvirt/variables.tf @@ -26,7 +26,7 @@ variable "ssh_connection" { } variable "libvirt_pool_path" { - default = "/etc/libvirt/k3s-paas-pool" + default = "/var/lib/libvirt-pools/k3s-paas-pool" } variable "node_hostname" { diff --git a/tf-modules-services/mongo/main.tf b/tf-modules-services/mongo/main.tf new file mode 100644 index 0000000..f07fc45 --- /dev/null +++ b/tf-modules-services/mongo/main.tf @@ -0,0 +1,38 @@ +resource "random_password" "mongo_root_password" { + length = 16 + special = true +} + +resource "helm_release" "mongodb" { + name = "mongo-db" + chart = "mongodb" + repository = "https://charts.bitnami.com/bitnami" + version = var.mongo_version + namespace = "mongo" + atomic = true + wait_for_jobs = true + timeout = 180 + create_namespace = true + + values = [ + <