Feature: Add a persistent backend to the cloud-init server which uses duckdb via quack/quack

.github/workflows/lint.yaml

@@ -0,0 +1,25 @@
+name: Run golangci-lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull requests. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: stable
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.1

.goreleaser-darwin.yaml

@@ -0,0 +1,42 @@
+version: 2
+project_name: cloud-init
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+    - go install github.com/swaggo/swag/cmd/swag@latest
+    - swag init -g cmd/cloud-init-server/main.go  
+
+builds:
+  - id: darwin
+    main: ./cmd/cloud-init-server
+    binary: cloud-init-server
+    # export GIT_STATE=$(if git diff-index --quiet HEAD --; then echo 'clean'; else echo 'dirty'; fi)
+    # export BUILD_HOST=$(hostname)
+    # export GO_VERSION=$(go version | awk '{print $3}')
+    # export BUILD_USER=$(whoami)
+    ldflags:
+      - "-X 'main.GitCommit={{.Commit}}' \
+         -X 'main.BuildTime={{.Timestamp}}' \
+         -X 'main.Version={{.Version}}' \
+         -X 'main.GitBranch={{.Branch}}' \
+         -X 'main.GitTag={{.Tag}}' \
+         -X 'main.GitState={{ .Env.GIT_STATE }}' \
+         -X 'main.BuildHost={{ .Env.BUILD_HOST }}' \
+         -X 'main.GoVersion={{ .Env.GO_VERSION }}' \
+         -X 'main.BuildUser={{ .Env.BUILD_USER }}'"
+    goos:
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+    goamd64:
+      - v3
+      - v1
+    env:
+      - CGO_ENABLED=1
+
+# The lines beneath this are called `modelines`. See `:help modeline`
+# Feel free to remove those if you don't want/use them.
+# yaml-language-server: $schema=https://raw.githubusercontent.com/goreleaser/goreleaser/v2.11.2/www/docs/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

.goreleaser.yaml

@@ -1,14 +1,14 @@
-
-version: 2.4
+version: 2
 project_name: cloud-init
 before:
   hooks:
     # You may remove this if you don't use go modules.
     - go mod tidy
+    - go install github.com/swaggo/swag/cmd/swag@latest
     - swag init -g cmd/cloud-init-server/main.go
 
 builds:
-  - id: cloud-init
+  - id: linux
     main: ./cmd/cloud-init-server
     binary: cloud-init-server
     # export GIT_STATE=$(if git diff-index --quiet HEAD --; then echo 'clean'; else echo 'dirty'; fi)
@@ -27,14 +27,17 @@ builds:
          -X 'main.BuildUser={{ .Env.BUILD_USER }}'"
     goos:
       - linux
-      - darwin
     goarch:
       - amd64
       - arm64
     goamd64:
       - v3
+      - v1
     env:
-      - CGO_ENABLED=0
+      - CGO_ENABLED=1
+      - CC={{ if eq .Arch "arm64" }}aarch64-linux-gnu-gcc{{ else }}gcc{{ end }}
+      - CXX={{ if eq .Arch "arm64" }}aarch64-linux-gnu-g++{{ else }}g++{{ end }}
+
 
 dockers:
   - image_templates:
@@ -125,5 +128,5 @@ changelog:
 
 # The lines beneath this are called `modelines`. See `:help modeline`
 # Feel free to remove those if you don't want/use them.
-# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# yaml-language-server: $schema=https://raw.githubusercontent.com/goreleaser/goreleaser/v2.11.2/www/docs/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile

@@ -1,30 +1,65 @@
+# syntax=docker/dockerfile:1.7
+FROM ubuntu:24.04 AS duckdb-ext
+ARG TARGETARCH
+ARG DUCKDB_VERSION="1.4.0"
 
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates curl unzip \
+ && rm -rf /var/lib/apt/lists/*
 
-FROM chainguard/wolfi-base:latest
+# Map TARGETARCH (docker buildx) or uname -m to DuckDB CLI artifact name
+# amd64|x86_64 => linux-amd64, arm64|aarch64 => linux-arm64 (v1.4.0 release assets)
+RUN ARCH="${TARGETARCH:-$(uname -m)}" \
+ && case "${ARCH}" in \
+      amd64|x86_64)  CLI_URL="https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-amd64.zip" ;; \
+      arm64|aarch64) CLI_URL="https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-arm64.zip" ;; \
+      *) echo "unsupported arch: ${ARCH}" && exit 1 ;; \
+    esac \
+ && curl -fsSL "$CLI_URL" -o /tmp/duckdb.zip \
+ && unzip /tmp/duckdb.zip -d /usr/local/bin \
+ && chmod +x /usr/local/bin/duckdb \
+ && rm -f /tmp/duckdb.zip
 
-RUN apk add --no-cache tini
+ENV DUCKDB_HOME=/duckdb_home
+RUN mkdir -p "$DUCKDB_HOME"
 
-# Include curl in the final image.
-RUN set -ex \
-    && apk update \
-    && apk add --no-cache curl tini wireguard-tools iputils iproute2 \
-    && rm -rf /var/cache/apk/*  \
-    && rm -rf /tmp/*
 
-STOPSIGNAL SIGTERM
+# Preinstall whatever you need
+RUN duckdb -c "INSTALL 'json';" \
+ && duckdb -c "INSTALL 'parquet';" 
 
-# Get the boot-script-service from the builder stage.
-COPY cloud-init-server /usr/local/bin/
+# -------- runtime image --------
+FROM ubuntu:24.04
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates wireguard-tools tini \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY cloud-init-server /usr/local/bin/cloud-init-server
+
+#Set the golang modcache
+ENV GOMODCACHE=/tmp/.cache
+RUN mkdir -p "$GOMODCACHE" && chown 65534:65534 "$GOMODCACHE"
+ENV XDG_CACHE_HOME=/tmp/.cache
+
+
+ENV DUCKDB_HOME=/opt/duckdbhome
+RUN mkdir -p "$DUCKDB_HOME" && chown 65534:65534 "$DUCKDB_HOME"
+
+# Copy the preinstalled extensions (built for the current arch)
+COPY --from=duckdb-ext /duckdb_home/ ${DUCKDB_HOME}/
+RUN chown -R 65534:65534 ${DUCKDB_HOME}
+
+# Store data in $DATADIR
+ENV DATADIR=/data
+RUN mkdir -p "${DATADIR}"
+VOLUME "$DATADIR"
+RUN chown -R 65534:65534 ${DATADIR}
 
 ENV TOKEN_URL="http://opaal:3333/token"
 ENV SMD_URL="http://smd:27779"
 ENV LISTEN="0.0.0.0:27777"
 
-
-# nobody 65534:65534
 USER 65534:65534
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["/usr/local/bin/cloud-init-server"]
 
-# Set up the command to start the service.
-CMD /usr/local/bin/cloud-init-server 
-
-ENTRYPOINT ["/sbin/tini", "--"]

build-in-container.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# This script uses the latest Ubuntu 24.04 container to build the project with GoReleaser.  It emulates the GitHub Actions environment as closely as possible.
+# Before submitting a PR for release/build. please run this script to ensure your PR will pass the build.
+
+# Name of the container
+CONTAINER_NAME="goreleaser-build"
+
+# Directory where built binaries will be available
+OUTPUT_DIR="$(pwd)/dist"
+
+export GIT_STATE=$(if git diff-index --quiet HEAD --; then echo 'clean'; else echo 'dirty'; fi)
+export BUILD_HOST=$(hostname)
+export GO_VERSION=$(go version | awk '{print $3}')
+export BUILD_USER=$(whoami)
+
+# Start a new disposable Ubuntu 24.04 container with the current directory mounted
+${CONTAINER_CMD:-docker} run --rm -it \
+    --name "$CONTAINER_NAME" \
+    -v "$(pwd)":/workspace \
+    -v ${CONTAINER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock \
+    -w /workspace \
+    ubuntu:24.04 bash -c "
+
+    # Suppress timezone prompts
+    export DEBIAN_FRONTEND=noninteractive
+    export TZ=UTC
+
+    # Update package lists and install dependencies
+    apt update && apt install -y git gcc g++ make \
+    ca-certificates curl gnupg \
+    gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu \
+    libc6-dev-arm64-cross software-properties-common \
+    clang-tools libstdc++-13-dev-arm64-cross
+
+    install -m 0755 -d /etc/apt/keyrings
+
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \
+    gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+
+    echo \
+    \"deb [arch=\$(dpkg --print-architecture) \
+    signed-by=/etc/apt/keyrings/docker.gpg] \
+    https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" | \
+    tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+    apt update && apt install -y \
+    docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+    # Install Go (match GitHub runner version)
+    curl -fsSL https://golang.org/dl/go1.21.5.linux-amd64.tar.gz | tar -C /usr/local -xz
+    export PATH=\$PATH:/usr/local/go/bin
+    go version  # Verify Go installation
+
+    # Set GOPATH and update PATH to include Go binaries
+    export GOPATH=\$(go env GOPATH)
+    export PATH=\$PATH:\$GOPATH/bin
+    echo \"GOPATH: \$GOPATH\" && echo \"PATH: \$PATH\"
+
+    # Install Goreleaser
+    curl -sL https://github.com/goreleaser/goreleaser/releases/latest/download/goreleaser_Linux_x86_64.tar.gz | tar -xz -C /usr/local/bin
+    goreleaser --version  # Verify Goreleaser installation
+
+    # Setup Docker buildx for multi-platform builds
+    docker buildx create --use --name goreleaser-builder || true
+    docker buildx inspect --bootstrap
+
+    # Set Build Environment Variables
+    export GIT_STATE="$GIT_STATE"
+    export BUILD_HOST="$BUILD_HOST"
+    export BUILD_USER="$BUILD_USER"
+    export GO_VERSION=$(go version | awk '{print $3}')
+
+    # Convince git that our directory is safe
+    git config --global --add safe.directory /workspace
+
+    # Run Goreleaser
+    goreleaser release --snapshot --clean --skip archive,publish
+"
+
+# Notify user of success
+echo "✅ Build complete! Check the output in: $OUTPUT_DIR"

cmd/cloud-init-server/group_handlers.go

@@ -18,7 +18,7 @@ import (
 //	@Produce		json
 //	@Success		200	{object}	map[string]cistore.ClusterDefaults
 //	@Failure		500	{object}	nil
-//	@Router			/cloud-init/admin/groups [get]
+//	@Router			/admin/groups [get]
 func (h CiHandler) GetGroups(w http.ResponseWriter, r *http.Request) {
 	var (
 		groups map[string]cistore.GroupData
@@ -58,7 +58,7 @@ func (h CiHandler) GetGroups(w http.ResponseWriter, r *http.Request) {
 //	@Failure		422		{object}	nil
 //	@Header			201		{string}	Location			"/groups/{id}"
 //	@Param			group	body		cistore.GroupData	true	"Group data"
-//	@Router			/cloud-init/admin/groups [post]
+//	@Router			/admin/groups [post]
 func (h CiHandler) AddGroupHandler(w http.ResponseWriter, r *http.Request) {
 	var (
 		data cistore.GroupData
@@ -91,14 +91,15 @@ func (h CiHandler) AddGroupHandler(w http.ResponseWriter, r *http.Request) {
 //	@Success		200	{object}	cistore.GroupData
 //	@Failure		500	{object}	nil
 //	@Param			id	path		string	true	"Group ID"
-//	@Router			/cloud-init/admin/groups/{id} [get]
+//	@Router			/admin/groups/{id} [get]
 func (h CiHandler) GetGroupHandler(w http.ResponseWriter, r *http.Request) {
 	var (
-		id    string = chi.URLParam(r, "id")
+		id    string
 		data  cistore.GroupData
 		bytes []byte
 		err   error
 	)
+	id = chi.URLParam(r, "id")
 
 	data, err = h.store.GetGroupData(id)
 	if err != nil {
@@ -111,7 +112,10 @@ func (h CiHandler) GetGroupHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	w.Write(bytes)
+	if _, err := w.Write(bytes); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 }
 
 // UpdateGroupHandler godoc
@@ -132,14 +136,16 @@ func (h CiHandler) GetGroupHandler(w http.ResponseWriter, r *http.Request) {
 //	@Header			201			{string}	Location			"/groups/{name}"
 //	@Param			name		path		string				true	"Group name"
 //	@Param			group_data	body		cistore.GroupData	true	"Group data"
-//	@Router			/cloud-init/admin/groups/{name} [put]
+//	@Router			/admin/groups/{name} [put]
 func (h CiHandler) UpdateGroupHandler(w http.ResponseWriter, r *http.Request) {
 	var (
-		groupName string = chi.URLParam(r, "name")
+		groupName string
 		data      cistore.GroupData
 		err       error
 	)
 
+	groupName = chi.URLParam(r, "name")
+
 	data, err = parseData(r)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusUnprocessableEntity)
@@ -164,12 +170,13 @@ func (h CiHandler) UpdateGroupHandler(w http.ResponseWriter, r *http.Request) {
 //	@Success		200	{object}	nil
 //	@Failure		500	{object}	nil
 //	@Param			id	path		string	true	"Group ID"
-//	@Router			/cloud-init/admin/groups/{id} [delete]
+//	@Router			/admin/groups/{id} [delete]
 func (h CiHandler) RemoveGroupHandler(w http.ResponseWriter, r *http.Request) {
 	var (
-		id  string = chi.URLParam(r, "id")
+		id  string
 		err error
 	)
+	id = chi.URLParam(r, "id")
 	err = h.store.RemoveGroupData(id)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)