mafyuh/iac
1
0
Fork 0
Code Issues 6 Pull requests 7 Projects 1 Wiki Activity Actions

Merge branch 'main' into renovate/ghcr.io-linuxserver-kasm
All checks were successful
Lint on PR / Lint YAML files (pull_request) Successful in 11s
Details

Browse source
This commit is contained in:
Matt Reeves 2024-07-31 20:52:22 -04:00
commit 3bb4522f07
16 changed files with 44 additions and 4295 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
.forgejo/workflows/CD.yml
View file

@ -14,6 +14,10 @@ jobs:
- name: Fetch all history for git diff
run: git fetch --depth=2
- name: Install jq
run: |
apt-get update && apt-get install -y jq
- name: Detect modified folders
id: detect-changes
run: |
@ -85,10 +89,13 @@ jobs:
sleep 45 # Delay for 45 seconds before fetching logs
job_id=$(curl -s -H "Authorization: Bearer ${{ secrets.AWX_API_TOKEN }}" "https://awx.mafyuh.xyz/api/v2/job_templates/13/jobs/?order_by=-id" | grep -oP '"id":\s*\K[0-9]+')
logs=$(curl -s -H "Authorization: Bearer ${{ secrets.AWX_API_TOKEN }}" "https://awx.mafyuh.xyz/api/v2/jobs/$job_id/stdout/?format=json")
job_id=$(curl -s -H "Authorization: Bearer ${{ secrets.AWX_API_TOKEN }}" https://awx.mafyuh.xyz/api/v2/job_templates/13/jobs/?order_by=-id | jq -r '.results[0].id')
logs=$(curl -s -H "Authorization: Bearer ${{ secrets.AWX_API_TOKEN }}" https://awx.mafyuh.xyz/api/v2/jobs/$job_id/stdout/?format=json)
echo "AWX Job Logs for folder: $folder"
echo "Range:"
echo "Start: $(echo "$logs" | jq -r '.range.start')"
echo "End: $(echo "$logs" | jq -r '.range.end')"
echo "Absolute End: $(echo "$logs" | jq -r '.range.absolute_end')"
echo "Content:"
echo "$logs" | grep -oP '"content":\s*"\K[^"]+' | sed 's/\\n/\n/g'
echo "$(echo "$logs" | jq -r '.content')"
done

4
.forgejo/workflows/yamllint.yml
View file

@ -17,6 +17,10 @@ jobs:
with:
node-version: 14
- name: Install yamllint
run: |
npm install -g yaml-lint
- name: Show yamllint version
run: |
yamllint --version

2
README.md
View file

@ -3,7 +3,7 @@ Currently migrating [Auto-Homelab](https://git.mafyuh.dev/mafyuh/Auto-Homelab),
ToDo
- [ ] Update Readme
- [ ] Migrate all containers to use this repo (DB migrations)
- [x] Migrate all containers to use this repo (DB migrations)
- [x] Get OpenTofu migrated to this repo
- [ ] Get Kubernetes repo migrated over
- [ ] Re-bootstrap Flux

4
docker/AI/docker-compose.yml
View file

@ -2,7 +2,7 @@ version: "3.8"
services:
ollama:
image: docker.mafyuh.xyz/ollama/ollama:0.2.6
image: ollama/ollama:0.3.1
container_name: ollama
restart: unless-stopped
volumes:
@ -29,7 +29,7 @@ services:
- host.docker.internal:host-gateway
mindsdb:
image: docker.mafyuh.xyz/mindsdb/mindsdb:v24.7.3.0
image: mindsdb/mindsdb:v24.7.5.0
container_name: mindsdb
ports:
- 47334:47334

14
docker/README.md
View file

@ -1,17 +1,15 @@
[![Yamllint](https://git.mafyuh.dev/mafyuh/Auto-Homelab/badges/workflows/yamllint.yml/badge.svg)](https://git.mafyuh.dev/mafyuh/Auto-Homelab/actions)
[![Yamllint](https://git.mafyuh.dev/mafyuh/Auto-Homelab/badges/workflows/CD.yml/badge.svg)](https://git.mafyuh.dev/mafyuh/Auto-Homelab/actions)
[![Yamllint](https://git.mafyuh.dev/mafyuh/iac/badges/workflows/yamllint.yml/badge.svg)](https://git.mafyuh.dev/mafyuh/iac/actions)
[![Yamllint](https://git.mafyuh.dev/mafyuh/iac/badges/workflows/CD.yml/badge.svg)](https://git.mafyuh.dev/mafyuh/iac/actions)
[![Renovate](https://git.mafyuh.dev/renovatebot/renovate/badges/workflows/renovate.yml/badge.svg)](https://git.mafyuh.dev/renovatebot/renovate/actions)
[![Pulls](https://git.mafyuh.dev/mafyuh/Auto-Homelab/badges/pulls.svg)](https://git.mafyuh.dev/mafyuh/Auto-Homelab/pulls)
[![Pulls](https://git.mafyuh.dev/mafyuh/iac/badges/pulls.svg)](https://git.mafyuh.dev/mafyuh/iac/pulls)
![Header Image](https://raw.githubusercontent.com/Mafyuh/homelab-svg-assets/main/assets/header_.png)
<div align="center">
# Auto-Homelab
Homelab docker-compose environment defined in code. Using Forgejo Actions and Renovate bot for CI, AWX Tower and Forgejo Actions for CD. This is how I keep my Homelab UTD.
</div>
<p align="center">
<a href="https://git.mafyuh.dev/mafyuh/Auto-Homelab/wiki">Wiki</a> |
<a href="https://git.mafyuh.dev/mafyuh/iac/wiki">Wiki</a> |
<a href="https://loganmarchione.com/2022/10/how-to-run-renovate-on-a-self-hosted-gitea-and-drone-instance/">How to Setup</a> |
<a href="https://mafyuh.com">Blog</a> |
<a href="https://www.youtube.com/watch?v=5CkCr9U_Q1Y">Inspiration</a> |
@ -306,7 +304,3 @@ graph TD
```
## To-Do
[View Project Board](https://git.mafyuh.dev/mafyuh/Auto-Homelab/projects/1)

2
docker/ag-main/docker-compose.yml
View file

@ -22,7 +22,7 @@ services:
- 5443:5443/udp
adguardhome-sync:
image: ghcr.io/linuxserver/adguardhome-sync@sha256:ee2ddd62c61d13e01da7f96135d1438376a22992c7d9951b2429bb2cfd64a05a
image: ghcr.io/linuxserver/adguardhome-sync@sha256:c6bad810acfc292b9220936751194f6ae9800b1228385ae8f2130fba280b79ee
container_name: adguardhome-sync
environment:
- PUID=1000

6
docker/arm/docker-compose.yml
View file

@ -1,7 +1,7 @@
---
services:
server:
image: codeberg.org/forgejo/forgejo:7.0.5
image: codeberg.org/forgejo/forgejo:8.0.0
container_name: forgejo
environment:
- USER_UID=1000
@ -107,7 +107,7 @@ services:
networks:
- gitea_main
n8n:
image: ghcr.io/n8n-io/n8n:1.51.0
image: ghcr.io/n8n-io/n8n:1.53.0
container_name: n8n
ports:
- 5678:5678
@ -137,7 +137,7 @@ services:
ipv4_address: 172.25.0.25
syncthing:
image: ghcr.io/linuxserver/syncthing@sha256:6fb87abe19730aad01b6511d660a0664be424986ae662b85eb76f0ce585c4890
image: ghcr.io/linuxserver/syncthing@sha256:84e9cd99d247d6ef31fc8c1a6967f068a8352a530095bb402bf3bb298aa10696
container_name: syncthing
hostname: ARM #optional
environment:

10
docker/arrs/docker-compose.yml
View file

@ -1,7 +1,7 @@
---
services:
bazarr:
image: ghcr.io/linuxserver/bazarr@sha256:25f0d19ba9226fdf15aec0fc7fa134ba89fc6f77f09d7bd45b33c5960c47821e
image: ghcr.io/linuxserver/bazarr@sha256:e70de8a714ac57395d45052392001ec433b8f48aa1c204f13dae312cbcbe43af
container_name: bazarr
ports:
- "6767:6767"
@ -15,7 +15,7 @@ services:
- PGID=1000
lidarr:
image: ghcr.io/linuxserver/lidarr@sha256:3c093803236a9f694f805392a5d6461b550f70b3f9cf4be0bc73b3428e3dbe90
image: ghcr.io/linuxserver/lidarr@sha256:5855582eee07024c2584c01aaf955901303162bf00de183a1fbf2e9b81041695
container_name: lidarr
ports:
- "8686:8686"
@ -31,7 +31,7 @@ services:
- PGID=1000
prowlarr:
image: ghcr.io/linuxserver/prowlarr@sha256:9a98da4fb506278ce92c2fdbb4e08a38418dd1456e3dd8b3a1f00a42a8ec4fb0
image: ghcr.io/linuxserver/prowlarr@sha256:7fe57565907f4f776d43c15b2e020a0e4a62fe1e04e80e25b85a3ae4ca49e5d0
container_name: prowlarr
ports:
- "9696:9696"
@ -137,10 +137,10 @@ services:
privileged: true
devices:
- /dev/kmsg
image: gcr.io/cadvisor/cadvisor:v0.49.1
image: gcr.io/cadvisor/cadvisor:v0.50.0
syncthing:
image: ghcr.io/linuxserver/syncthing@sha256:6fb87abe19730aad01b6511d660a0664be424986ae662b85eb76f0ce585c4890
image: ghcr.io/linuxserver/syncthing@sha256:84e9cd99d247d6ef31fc8c1a6967f068a8352a530095bb402bf3bb298aa10696
container_name: syncthing
hostname: ARRS
environment:

4
docker/authentik/docker-compose.yml
View file

@ -32,7 +32,7 @@ services:
volumes:
- redis:/data
server:
image: ghcr.io/goauthentik/server@sha256:f79b6cf35b29a8591136afb633ddd46da303e9cd2ca82c7cdfbd020a780e15e8
image: ghcr.io/goauthentik/server@sha256:a98f95518269d01bb812eb0e12c6647f3d08a36e37b9fdbdccf9345d4431b9f0
restart: unless-stopped
command: server
environment:
@ -53,7 +53,7 @@ services:
- postgresql
- redis
worker:
image: ghcr.io/goauthentik/server@sha256:f79b6cf35b29a8591136afb633ddd46da303e9cd2ca82c7cdfbd020a780e15e8
image: ghcr.io/goauthentik/server@sha256:a98f95518269d01bb812eb0e12c6647f3d08a36e37b9fdbdccf9345d4431b9f0
restart: unless-stopped
command: worker
environment:

2
docker/downloaders/docker-compose.yml
View file

@ -69,7 +69,7 @@ services:
privileged: true
devices:
- /dev/kmsg
image: gcr.io/cadvisor/cadvisor:v0.49.1
image: gcr.io/cadvisor/cadvisor:v0.50.0
node-exporter:
image: docker.mafyuh.xyz/prom/node-exporter:v1.8.2

4
docker/jellyfin/docker-compose.yml
View file

@ -1,7 +1,7 @@
---
services:
jellyfin:
image: ghcr.io/linuxserver/jellyfin@sha256:843878f35dc700e502798b39edb24b84b7b7ff0788f1b7b9b1e71d4f8d34f951
image: ghcr.io/linuxserver/jellyfin@sha256:eadf16cadd823a5cbe1b92750ee74111f9b2cac894834477ab4a43a5fc835ebe
container_name: jellyfin
devices:
- /dev/dri/renderD129:/dev/dri/renderD129
@ -35,7 +35,7 @@ services:
privileged: true
syncthing:
image: ghcr.io/linuxserver/syncthing@sha256:6fb87abe19730aad01b6511d660a0664be424986ae662b85eb76f0ce585c4890
image: ghcr.io/linuxserver/syncthing@sha256:84e9cd99d247d6ef31fc8c1a6967f068a8352a530095bb402bf3bb298aa10696
container_name: syncthing
hostname: JF
environment:

2
docker/netboot/docker-compose.yml
View file

@ -1,7 +1,7 @@
---
services:
netbootxyz:
image: ghcr.io/linuxserver/netbootxyz@sha256:3e7af245944519374e7e01422024f3b5baf67fa252ec92390fee1d57cd6cc1f6
image: ghcr.io/linuxserver/netbootxyz@sha256:8adbd502fa2f31f03f39b35c511e3484b1a7100ee7bc4ab878518066881079e9
container_name: netboot
environment:
- PUID=1000

21
docker/pages/docker-compose.yml
View file

@ -1,21 +0,0 @@
---
services:
pages:
image: codeberg.org/codeberg/pages-server:v5.1
container_name: codeberg-pages
environment:
- GITEA_ROOT=https://git.mafyuh.dev
- GITEA_API_TOKEN=$GITEA_API_TOKEN
- ACME_ACCEPT_TERMS=true
- CLOUDFLARE_EMAIL=$CLOUDFLARE_EMAIL
- CLOUDFLARE_API_KEY=$CLOUDFLARE_API_KEY
- DNS_PROVIDER=cloudflare
- PAGES_DOMAIN=mafyuh.co
- CF_ZONE_API_TOKEN=$CF_ZONE_API_TOKEN
- CF_DNS_API_TOKEN=$CF_DNS_API_TOKEN
- ENABLE_HTTP_SERVER=false
ports:
- "80:80"
- "443:443"
volumes:
- /home/ubuntu/pages/datanew:/data

4167
docker/scripts/CD.json
View file

File diff suppressed because it is too large Load diff

74
docker/scripts/README.md
View file

@ -1,74 +0,0 @@
# n8n Continuous Deployment
Previously used this script for CD, it worked, but I wanted an instant deployment and not just ran on a schedule. So I created an n8n automation. Here's the visual breakdown:
```mermaid
graph LR
A((Webhook Received)) --> B{PR Merged?}
B -- No --> C(End)
B -- Yes --> D{Extract App Name}
D --> E{SSH to Host Machine}
E --> F{Git Pull & Docker Compose Up}
F --> G{Error?}
G -- Yes --> H[Send Gotify Notification]
G -- No --> I[Extract PR Number]
I --> J[Add PR Number to Forgejo API URL]
J --> K[Send Review with Logs]
K --> L(End)
H --> L
```
## Outline
- Webhook is sent whenever a PR is merged
- Filters out any new, edited or re-opened PR's
- Checks the PR title for app name, which is filtered so each SSH session matches based on the IF conditions.
- SSH in to the machine that hosts that docker compose stack running git pull and docker compose up -d
- Optional, If there is any error message we sent Gotify a link to the PR and error message
- Set number from JSON to number in n8n
- Add that number to URL of API request to Forgejo which we add a review with the logs if no errors. https://git.mafyuh.dev/mafyuh/Auto-Homelab/pulls/222#issuecomment-1799
The only exception is n8n itself. Which for now I will manually update.
To import this into your n8n, create a new workflow, top right click 3 dots - Import from URL and paste https://git.mafyuh.dev/mafyuh/Auto-Homelab/raw/branch/main/scripts/CD.json
## dccd instructions (no longer used besides ag-backup)
Modified version of https://github.com/loganmarchione/dccd, with Gotify notification on error and allowing to choose which directory is cd'd into before running script. I run this on each docker host through crontab every 30 minutes, just adding the -f flag with the folder which that host represents. So each directory represents a host
Clone this repo:
```
git clone https://git.mafyuh.dev/mafyuh/Auto-Homelab.git
```
Make executable:
```
sudo chmod +x /home/mafyuh/Auto-Homelab/scripts/dccd.sh
```
Example:
```
./dccd.sh -b main -d /home/mafyuh/Auto-Homelab -p -f 'arrs'
```
Crontab every 30 mins:
```
*/30 * * * * /home/mafyuh/Auto-Homelab/scripts/dccd.sh -b main -d /home/mafyuh/Auto-Homelab -l /tmp/dccd.txt -p -f 'arrs'
```
View Logs:
```
cat /tmp/dccd.txt
```
Export Variables:
```
export GOTIFY_BASE_URL="https:go.example.com"
export GOTIFY_TOKEN="token"
```
Full Usage:
```
Options:
-b <name> Specify the remote branch to track (default: main)
-d <path> Specify the base directory of the git repository (required)
-h Show this help message
-l <path> Specify the path to the log file (default: /tmp/dccd.log)
-p Specify if you want to prune docker images (default: don't prune)
-x <path> Exclude directories matching the specified pattern (relative to the base directory)
-f <pattern> Specify the pattern for folder names to match
```

6
terraform/cloud-init.tf
View file

@ -28,6 +28,10 @@ resource "proxmox_virtual_environment_file" "cloud_config" {
- systemctl start qemu-guest-agent
- curl -fsSL https://get.docker.com | sudo sh
- su - mafyuh -c 'git clone https://git.mafyuh.dev/mafyuh/iac.git /home/mafyuh/iac'
- su - mafyuh -c 'git config --global user.name "Mafyuh"'
- su - mafyuh -c 'git config --global user.email "matt@mafyuh.com"'
- su - mafyuh -c 'echo "alias dcu=\'docker compose up -d\'" >> /home/mafyuh/.bashrc
- su - mafyuh -c 'echo "alias dcd=\'docker compose down\'" >> /home/mafyuh/.bashrc
- echo "done" > /tmp/cloud-config.done
EOF
@ -63,6 +67,8 @@ resource "proxmox_virtual_environment_file" "cloud_config2" {
- su - mafyuh -c 'git clone https://git.mafyuh.dev/mafyuh/iac.git /home/mafyuh/iac'
- su - mafyuh -c 'git config --global user.name "Mafyuh"'
- su - mafyuh -c 'git config --global user.email "matt@mafyuh.com"'
- su - mafyuh -c 'echo "alias dcu=\'docker compose up -d\'" >> /home/mafyuh/.bashrc
- su - mafyuh -c 'echo "alias dcd=\'docker compose down\'" >> /home/mafyuh/.bashrc
- echo "done" > /tmp/cloud-config.done
EOF