cli: expand GitHub owner/repo shorthand in add-repo

.github/workflows/main.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/tests.yml

@@ -0,0 +1,33 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-26.04
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: recursive
+    - name: Install Meson
+      shell: bash
+      run: |
+          sudo apt update
+          sudo apt install gcc-15 g++-15 python3 python3-pip python3-setuptools python3-wheel ninja-build
+          pip3 install meson
+    - name: Build
+      shell: bash
+      run: |
+            meson setup --reconfigure --buildtype=release builddir
+            cd builddir
+            meson test
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: testlog.txt
+        path: builddir/meson-logs/testlog.txt

.gitignore

@@ -2,4 +2,5 @@ builddir/
 docs/
 subprojects/*
 !subprojects/*.wrap
+!subprojects/packagefiles
 build/

cli/main.c

@@ -110,14 +110,25 @@ int main(int argc, char* argv[])
         Repository repository;
         for (int i = command_index+1; i < argc; i++)
         {
-            if ((error = KPM_AddRepository(&kpm, argv[i], &repository, &kpm_io)) != KPM_OK)
-                kpm_io.log(KPM_VERBOSITY_ERROR, "Could not add repository '%s' (%i: %s)", argv[i], error, KPM_ErrorToString(error));
+            char* url;
+            char* slash = strchr(argv[i], '/');
+            if (strpbrk(argv[i], ".:") == NULL && slash != NULL)
+            {
+                char* path = strchr(slash+1, '/');
+                int repo_len = path != NULL ? path - argv[i] : strlen(argv[i]);
+                url = asprintf_hd("https://raw.githubusercontent.com/%.*s/HEAD%s.json", repo_len, argv[i], path != NULL ? path : "/manifest");
+            }
+            else
+                url = strdup(argv[i]);
+
+            if ((error = KPM_AddRepository(&kpm, url, &repository, &kpm_io)) != KPM_OK)
+                kpm_io.log(KPM_VERBOSITY_ERROR, "Could not add repository '%s' (%i: %s)", url, error, KPM_ErrorToString(error));
             else
             {
                 kpm_io.log(KPM_VERBOSITY_INFO, "Added repository '%s'", repository.id);
                 KPM_FreeRepository(&repository);
             }
-
+            free(url);
         }
 
         if (error == KPM_OK)
@@ -380,7 +391,7 @@ kpm_io.log(KPM_VERBOSITY_INFO, "usage: kpm [--help, -h] [--fbink] [-y] {version
 version:\n\
     Returns the current version of the KPM library and CLI\n\
 add-repo:\n\
-    Add a repository from one or more URLs\n\
+    Add a repository from one or more URLs, or GitHub owner/repo[/path] shorthands (path defaults to manifest, .json is appended)\n\
 remove-repo:\n\
     Remove one or more repositories given one or more IDs\n\
 list-repo:\n\

cli/meson.build

@@ -10,6 +10,16 @@ if KPM_PLATFORM == ''
     KPM_PLATFORM = host_machine.cpu_family()
 endif
 
-subproject('fbink', default_options: ['bitmap=enabled', 'button-scan=disabled', 'fonts=disabled', 'image=disabled', 'inputlib=disabled', 'opentype=disabled', 'fbdepth=disabled', 'target=' + meson.get_external_property('target', 'Linux')])
+subproject('fbink', default_options: [
+    'bitmap=enabled',
+    'button-scan=disabled',
+    'fonts=disabled',
+    'image=disabled',
+    'inputlib=disabled',
+    'opentype=disabled',
+    'fbink=disabled',
+    'fbdepth=disabled',
+    'target=' + meson.get_external_property('target', 'Linux')
+])
 fbink_dep = dependency('fbink')
 executable('kpm', sources, include_directories: include_dir, dependencies: [kpm_dep, fbink_dep], build_rpath: kmc_lib_dir, install_rpath: kmc_lib_dir)

kpm-helper.py

@@ -13,7 +13,7 @@
 logging.basicConfig(format="[%(levelname)s] %(message)s", level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-KPM_MANIFEST_VERSION = 2
+KPM_MANIFEST_VERSION = 3
 valid_supported_platforms = [
     "kindle",
     "kindle5",
@@ -103,9 +103,9 @@ def __init__(
 
         # @TODO
         if manifest_version != KPM_MANIFEST_VERSION:
-            if manifest_version["manifest_version"] == 1:
+            if manifest_version["manifest_version"] < KPM_MANIFEST_VERSION:
                 logger.warning(
-                    "Manifest v1 is deprecated. Please upgrade to manifest v2."
+                    f"Manifest v{manifest_version['manifest_version']} is deprecated. Please upgrade to manifest v{KPM_MANIFEST_VERSION}."
                 )
             else:
                 logger.error(
@@ -248,6 +248,8 @@ def pack(self, output_path: str, compression: int = 5):
             file = tarfile.open(packageFilename, "w:")
         elif self.manifest_version >= 2:
             file = tarfile.open(packageFilename, "w:gz", compresslevel=compression)
+        elif self.manifest_version >= 3:
+            file = tarfile.open(packageFilename, "w:zst", level=compression)
         else:
             file = tarfile.open(
                 packageFilename, "w:xz", preset=compression

meson.build

@@ -11,19 +11,32 @@ add_global_arguments('-DKPM_PKG_PATH="' + get_option('pkg_path') + '"', language
 
 kmc_lib_dir = '/var/local/kmc/lib'
 
+subproject('zstd', default_options: [
+  'default_library=static',
+  'legacy_level=7',
+  'debug_level=0',
+  'static_runtime=true',
+  'bin_programs=false',
+  'bin_tests=false',
+  'bin_contrib=false',
+  'zlib=enabled',
+  'lzma=disabled',
+  'lz4=disabled',
+  'c_args=-lrt',
+  'cpp_args=-lrt'
+])
 subproject('liblzma', default_options: ['default_library=static'])
-subproject('zlib', default_options: ['default_library=static', 'tests=disabled'])
 subproject('sqlite3', default_options: ['default_library=static']) # PW2 sqlite3 is unusable it straight up doesn't compile I need to fix this
 subproject('cjson', default_options: ['tests=false', 'default_library=static'])
 subproject('libarchive', default_options: [
   'default_library=static',
   'openssl=disabled',
-  'zlib=enabled',
+  'zlib=disabled',
   'bz2lib=disabled',
   'libb2=disabled',
   'iconv=disabled',
   'lz4=disabled',
-  'zstd=disabled',
+  'zstd=enabled',
   'lzma=enabled',
   'lzo2=disabled',
   'cng=disabled',
@@ -35,7 +48,7 @@ subproject('libarchive', default_options: [
 
 ])
 subproject('google-brotli', default_options: ['default_library=static']) # Needed for cURL brotli support
-subproject('openssl', default_options: ['default_library=static', 'build_cli=false']) # Needed for cURL HTTPS support
+openssl = subproject('openssl', default_options: ['default_library=static', 'build_cli=false']) # Needed for cURL HTTPS support
 subproject('nghttp2', default_options: ['default_library=static']) # Needed for cURL HTTP/2 support
 curl = subproject('curl', default_options: [
   'tool=disabled',
@@ -85,6 +98,7 @@ sqlite3_dep = dependency('sqlite3', static: true, default_options: [
 curl_dep = curl.get_variable('curl_dep') # Get static libcurl
 libcjson_dep = dependency('libcjson', static: true)
 libarchive_dep = dependency('libarchive', static: true)
+libcrypto_dep = dependency('libcrypto', static: true)
 
 subdir('./stubs')
 subdir('./src')

src/include/internal_utils.h

@@ -7,4 +7,5 @@ void mkdir_r(char* path, __mode_t mode);
 void rmdir_r(char* path);
 
 char* vasprintf_hd(const char* format, va_list args);
-char* asprintf_hd(const char* format, ...) __attribute__((format(printf, 1, 2)));
+char* asprintf_hd(const char* format, ...) __attribute__((format(printf, 1, 2)));
+char* get_realpath(const char* location);

src/include/kpm/kpkg.h

@@ -0,0 +1,85 @@
+#pragma once
+
+#include "kpm/semver.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#define KPKG_MAGIC "KPKG"
+#define KPKG_MAGIC_LENGTH 4
+#define KPKG_VERSION 3
+
+typedef enum {
+    KPKG_OK,
+} KPKG_Result;
+
+typedef enum {
+    KINDLEANY = 0,
+    KINDLE = 1 << 0,
+    KINDLE5 = 1 << 1,
+    KINDLEPW2 = 1 << 2,
+    KINDLEHF = 1 << 3
+} KPKG_SupportedPlatforms;
+
+struct KPKG_Header
+{
+    char magic[KPKG_MAGIC_LENGTH]; // "KPKG"
+    uint8_t version; // Format version
+    SemVer package_version; // Version of the package
+    uint8_t supported_platforms; // Bitfield of KPKG_SupportedPlatforms
+    uint32_t dependency_count;
+    uint32_t file_count;
+    uint64_t strings_size; // String section size
+};
+typedef struct KPKG_Header KPKG_Header;
+
+struct KPKG_DependencyEntry
+{
+    uint64_t id; // Pointer in string section
+    SemVer min_version;
+    SemVer max_version;
+};
+typedef struct KPKG_DependencyEntry KPKG_DependencyEntry;
+
+struct KPKG_FileEntry
+{
+    uint64_t path; // Pointer in string section
+    uint8_t checksum[16];
+    uint32_t file_mode;
+    uint64_t offset; // Pointer in file data section
+};
+typedef struct KPKG_FileEntry KPKG_FileEntry;
+
+struct KPKG_File
+{
+    char* location; // The URL/path to the file
+    void* internal_file; // The memory mapped file or NULL
+    size_t map_size; // If file is local and memory mapped, this will indicate the length
+    KPKG_Header* header; // Local storage of header (always present)
+    KPKG_DependencyEntry* dependencies;
+    KPKG_FileEntry* files;
+    char* strings;
+};
+typedef struct KPKG_File KPKG_File;
+
+/**
+ * @brief Load a KPKG_File from the specified location
+ * 
+ * @param location The URL or path to the KPKG file
+ * @return KPKG_File* A pointer to the KPKG_File file object if valid, otherwise NULL
+ */
+KPKG_File* KPKG_LoadFile(const char* location);
+
+void KPKG_CloseFile(KPKG_File* file);
+
+/**
+ * @brief Validate a KPKG_File object such that none of the pointers are OOB, note: this function assumes that the entry and string arrays are not malformed.
+ * 
+ * @param kpkg_file The file object to validate
+ * @return true 
+ * @return false 
+ */
+bool KPKG_ValidateFile(KPKG_File* kpkg_file);
+
+char* KPKG_GetString(KPKG_File* kpkg_file, uint64_t offset);
+KPKG_DependencyEntry* KPKG_GetDependencies(KPKG_File* kpkg_file, size_t* count);
+KPKG_FileEntry* KPKG_GetFileEntries(KPKG_File* kpkg_file, size_t* count);

src/include/kpm/kpm.h

@@ -12,16 +12,15 @@
  */
 
 #include <sqlite3.h>
-#include <stddef.h>
 #include <sys/types.h>
 
 #include "semver.h"
 
-#define KPM_MANIFEST_VERSION 2
+#define KPM_MANIFEST_VERSION 3
 
 #define KPM_VERSION_MAJOR 0
-#define KPM_VERSION_MINOR 2
-#define KPM_VERSION_PATCH 3
+#define KPM_VERSION_MINOR 3
+#define KPM_VERSION_PATCH 0
 
 typedef enum
 {

src/include/kpm/semver.h

@@ -1,15 +1,16 @@
 #pragma once
 
 #include <limits.h>
+#include <stdint.h>
 
 #define VERSION_MAX UINT_MAX
 #define SEMVER_MAX { .major = UINT_MAX, .minor = UINT_MAX, .patch = UINT_MAX }
 
 struct SemVer
 {
-    unsigned int major;
-    unsigned int minor;
-    unsigned int patch;
+    uint32_t major;
+    uint32_t minor;
+    uint32_t patch;
 };
 typedef struct SemVer SemVer;
 

src/include/simpleGET.h

@@ -4,15 +4,35 @@
 
 struct SimpleGETRequest
 {
-    CURL* curl;
-    long response_code;
-    char* buffer;
-    size_t size;
+    CURL* curl; /**< The internal CURL object backing this request */
+    long response_code; /**< The received response code */
+    unsigned char* buffer; /**< The buffer of data from this request */
+    size_t size; /**< The current size of the buffer */
+    size_t max_size; /**< The maximum size of the response before the request is terminated */
 };
 typedef struct SimpleGETRequest SimpleGETRequest;
 
 size_t SimpleGET_Callback(char* ptr, size_t size, size_t nmemb, void* userdata);
-void SimpleGET_Initialise(SimpleGETRequest* request, const char* url);
+
+/**
+ * @brief Create a SimpleGETRequest
+ * 
+ * @param url The URL to request
+ * @return SimpleGETRequest* A pointer to the created SimpleGETRequest object
+ */
+SimpleGETRequest* SimpleGET_Initialise(const char* url);
+
+/**
+ * @brief Frees a SimpleGETRequest object
+ * 
+ * @param request The SimpleGetRequest object to free
+ */
 void SimpleGET_Cleanup(SimpleGETRequest* request);
-CURLcode SimpleGET_Perform(SimpleGETRequest* request);
-long SimpleGET_GetResponseCode(SimpleGETRequest* request);
+
+/**
+ * @brief Perform the request
+ * 
+ * @param request The SimpleGETRequest to perform
+ * @return CURLcode The internal CURL result of the request, given by curl_easy_perform
+ */
+CURLcode SimpleGET_Perform(SimpleGETRequest* request);