Add support for custom registries and repositories, read secrets from

clib_secrets.json

Author: nouwaarom

Makefile

@@ -12,20 +12,22 @@ RM      = rm -f
 MKDIR   = mkdir -p
 
 SRC  = $(wildcard src/*.c)
-COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c)
-ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h test/package/*.c test/cache/*.c)
+COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*)
+ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h src/repository/*.h src/repository/*.c test/package/*.c test/cache/*.c)
 SDEPS = $(wildcard deps/*/*.c)
 ODEPS = $(SDEPS:.c=.o)
 DEPS = $(filter-out $(ODEPS), $(SDEPS))
 OBJS = $(DEPS:.c=.o)
 
 export CC
 
+CFLAGS  += -std=gnu17 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -g -Wall -Werror=return-type -Wno-unused-function $(shell curl-config --cflags)
+
 ifdef STATIC
-	CFLAGS  += -DCURL_STATICLIB -std=c99 -Ideps -Wall -Werror=return-type -Wno-unused-function -U__STRICT_ANSI__ $(shell deps/curl/bin/curl-config --cflags)
+	CFLAGS  += -DCURL_STATICLIB $(shell deps/curl/bin/curl-config --cflags)
 	LDFLAGS += -static $(shell deps/curl/bin/curl-config --static-libs)
 else
-	CFLAGS  += -std=gnu17 -Ideps -g -Wall -Werror=return-type -Wno-unused-function $(shell curl-config --cflags)
+	CFLAGS  += $(shell curl-config --cflags)
 	LDFLAGS += $(shell curl-config --libs)
 endif
 

clib.json

@@ -44,7 +44,7 @@
     "stephenmathieson/gumbo-get-element-by-id.c": "*",
     "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*",
     "clibs/list": "*",
-    "jwerle/url.h": "0.0.*"
+    "jwerle/url.h": "0.1.*"
   },
   "development": {
     "stephenmathieson/describe.h": "2.0.1"

deps/http-get/http-get.c

@@ -97,7 +97,7 @@ static size_t http_get_file_cb(void *ptr, size_t size, size_t nmemb, void *strea
  * Request `url` and save to `file`
  */
 
-int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
+int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** headers, int header_count) {
   CURL *req = curl_easy_init();
   if (!req) return -1;
 
@@ -108,6 +108,15 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
     curl_easy_setopt(req, CURLOPT_SHARE, share);
   }
 
+  if (header_count > 0) {
+    struct curl_slist *chunk = NULL;
+    for (int i = 0; i < header_count; i++) {
+      chunk = curl_slist_append(chunk, headers[i]);
+    }
+    /* set our custom set of headers */
+    curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk);
+  }
+
   curl_easy_setopt(req, CURLOPT_URL, url);
   curl_easy_setopt(req, CURLOPT_HTTPGET, 1);
   curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1);
@@ -125,7 +134,7 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
 }
 
 int http_get_file(const char *url, const char *file) {
-  return http_get_file_shared(url, file, NULL);
+  return http_get_file_shared(url, file, NULL, NULL, 0);
 }
 
 /**

deps/http-get/http-get.h

@@ -25,7 +25,7 @@ http_get_response_t *http_get(const char *url, const char** headers, int header_
 http_get_response_t *http_get_shared(const char *url, void *, const char** headers, int header_count);
 
 int http_get_file(const char *, const char *);
-int http_get_file_shared(const char *, const char *, void *);
+int http_get_file_shared(const char *, const char *, void *, const char** headers, int header_count);
 
 void http_get_free(http_get_response_t *);
 

deps/url/url.h

@@ -51,6 +51,7 @@
  * http://en.wikipedia.org/wiki/URI_scheme
  */
 
+#ifdef URL_H_IMPLEMENTATION
 char *URL_SCHEMES[] = {
         // official IANA registered schemes
         "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw",
@@ -72,6 +73,7 @@ char *URL_SCHEMES[] = {
         // unofficial schemes
         "javascript", "jdbc", "doi"
 };
+#endif
 
 
 /**
@@ -102,7 +104,7 @@ typedef struct url_data {
  */
 
 url_data_t *
-url_parse (char *url);
+url_parse (const char *url);
 
 char *
 url_get_protocol (char *url);
@@ -151,7 +153,7 @@ url_data_inspect (url_data_t *data);
 
 
 // implementation
-
+#ifdef URL_H_IMPLEMENTATION
 
 // non C99 standard functions
 #if _POSIX_C_SOURCE < 200809L
@@ -220,7 +222,7 @@ get_part (char *url, const char *format, int l) {
 }
 
 url_data_t *
-url_parse (char *url) {
+url_parse (const char *url) {
     url_data_t *data = malloc(sizeof(url_data_t));
     if (!data) return NULL;
 
@@ -590,5 +592,6 @@ url_free (url_data_t *data) {
     free(data);
 }
 
+#endif
 
 #endif

src/clib-build.c

@@ -199,7 +199,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) {
 #ifdef DEBUG
     package = clib_package_new_from_slug(dir, 1);
 #else
-    package = clib_package_new_from_slug(dir, 0);
+    package = clib_package_new_from_slug_and_url(dir, 0);
 #endif
   }
 
@@ -330,7 +330,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) {
       char *dep_dir = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, 0);
       if (opts.dir && dependency && dependency->name) {
         dep_dir = path_join(opts.dir, dependency->name);
       }
@@ -401,7 +401,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) {
       char *slug = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, 0);
       char *dep_dir = path_join(opts.dir, dependency->name);
 
       free(slug);

src/clib-configure.c

@@ -192,7 +192,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) {
 #ifdef DEBUG
     package = clib_package_new_from_slug(dir, 1);
 #else
-    package = clib_package_new_from_slug(dir, 0);
+    package = clib_package_new_from_slug_and_url(dir, 0);
 #endif
   }
 
@@ -288,7 +288,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) {
       char *slug = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, 0);
       char *dep_dir = path_join(opts.dir, dependency->name);
 
       free(slug);
@@ -359,7 +359,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) {
       char *slug = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, 0);
       char *dep_dir = path_join(opts.dir, dependency->name);
 
       free(slug);

src/clib-install.c

@@ -17,9 +17,12 @@
 #include "parson/parson.h"
 #include "str-replace/str-replace.h"
 #include "version.h"
+#include <clib-secrets.h>
 #include <curl/curl.h>
 #include <libgen.h>
 #include <limits.h>
+#include <registry-manager.h>
+#include <repository.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -251,21 +254,6 @@ static int install_package(const char *slug) {
   long path_max = 4096;
 #endif
 
-  if (!root_package) {
-    const char *name = NULL;
-    char *json = NULL;
-    unsigned int i = 0;
-
-    do {
-      name = manifest_names[i];
-      json = fs_read(name);
-    } while (NULL != manifest_names[++i] && !json);
-
-    if (json) {
-      root_package = clib_package_new(json, opts.verbose);
-    }
-  }
-
   if ('.' == slug[0]) {
     if (1 == strlen(slug) || ('/' == slug[1] && 2 == strlen(slug))) {
       char dir[path_max];
@@ -291,10 +279,21 @@ static int install_package(const char *slug) {
     }
   }
 
-  if (!pkg) {
-    pkg = clib_package_new_from_slug(slug, opts.verbose);
+  // Read local config files.
+  clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json");
+  repository_init(secrets); // The repository requires the secrets for authentication.
+  clib_package_t *package = clib_package_load_local_manifest(0);
+
+  registries_t registries = registry_manager_init_registries(package->registries, secrets);
+  registry_manager_fetch_registries(registries);
+  wiki_package_ptr_t package_info = registry_manger_find_package(registries, slug);
+  if (!package_info) {
+    debug(&debugger, "Package %s not found in any registry.", slug);
+    return -1;
   }
 
+
+  pkg = clib_package_new_from_slug_and_url(slug, wiki_package_get_href(package_info), opts.verbose);
   if (NULL == pkg)
     return -1;
 
@@ -437,6 +436,21 @@ int main(int argc, char *argv[]) {
 
   clib_package_set_opts(package_opts);
 
+  if (!root_package) {
+    const char *name = NULL;
+    char *json = NULL;
+    unsigned int i = 0;
+
+    do {
+      name = manifest_names[i];
+      json = fs_read(name);
+    } while (NULL != manifest_names[++i] && !json);
+
+    if (json) {
+      root_package = clib_package_new(json, opts.verbose);
+    }
+  }
+
   int code = 0 == program.argc ? install_local_packages()
                                : install_packages(program.argc, program.argv);
 

src/clib-search.c

@@ -17,14 +17,17 @@
 #include "http-get/http-get.h"
 #include "logger/logger.h"
 #include "parson/parson.h"
+#include "registry-manager.h"
+#include "registry/wiki-registry.h"
 #include "strdup/strdup.h"
 #include "tempdir/tempdir.h"
 #include "version.h"
-#include "registry/wiki-registry.h"
+#include <clib-secrets.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+#include <url/url.h>
 
 #define CLIB_WIKI_URL "https://github.com/clibs/clib/wiki/Packages"
 
@@ -72,13 +75,13 @@ static int matches(int count, char *args[], wiki_package_ptr_t pkg) {
     char *href = NULL;
     int rc = 0;
 
-    name = clib_package_parse_name(wiki_package_get_repo(pkg));
+    name = clib_package_parse_name(wiki_package_get_id(pkg));
     COMPARE(name);
 
     description = strdup(wiki_package_get_description(pkg));
     COMPARE(description);
 
-    repo = strdup(wiki_package_get_repo(pkg));
+    repo = strdup(wiki_package_get_id(pkg));
     COMPARE(repo);
 
     href = strdup(wiki_package_get_href(pkg));
@@ -123,7 +126,7 @@ static char *wiki_html_cache() {
 static void display_package(const wiki_package_ptr_t pkg,
                             cc_color_t fg_color_highlight,
                             cc_color_t fg_color_text) {
-    cc_fprintf(fg_color_highlight, stdout, "  %s\n", wiki_package_get_repo(pkg));
+    cc_fprintf(fg_color_highlight, stdout, "  %s\n", wiki_package_get_id(pkg));
     printf("  url: ");
     cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_href(pkg));
     printf("  desc: ");
@@ -136,7 +139,7 @@ static void add_package_to_json(const wiki_package_ptr_t pkg,
     JSON_Value *json_pkg_root = json_value_init_object();
     JSON_Object *json_pkg = json_value_get_object(json_pkg_root);
 
-    json_object_set_string(json_pkg, "repo", wiki_package_get_repo(pkg));
+    json_object_set_string(json_pkg, "repo", wiki_package_get_id(pkg));
     json_object_set_string(json_pkg, "href", wiki_package_get_href(pkg));
     json_object_set_string(json_pkg, "description", wiki_package_get_description(pkg));
     json_object_set_string(json_pkg, "category", wiki_package_get_category(pkg));
@@ -176,26 +179,11 @@ int main(int argc, char *argv[]) {
 
     // We search the local manifest for extra registries.
     // It is important to give the extra registries preference over the default registry.
-    clib_package_t *package = clib_package_load_local_manifest(0);
-    wiki_registry_ptr_t* registries = malloc((package->registries->len + 1) * sizeof(wiki_registry_ptr_t));
-    int registry_count = 0;
-
-    list_iterator_t *registry_iterator = list_iterator_new(package->registries, LIST_HEAD);
-    list_node_t *node;
-    while ((node = list_iterator_next(registry_iterator))) {
-        registries[registry_count] = wiki_registry_create(node->val);
-        if (!wiki_registry_fetch(registries[registry_count])) {
-            printf("SEARCH, could not list packages from. %s\n", wiki_registry_get_url(registries[registry_count]));
-        } else {
-            registry_count++;
-        }
-    }
-    list_iterator_destroy(registry_iterator);
+    clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json");
 
-    // And add the default registry.
-    //registries[registry_count] = wiki_registry_create(CLIB_WIKI_URL);
-    //wiki_registry_fetch(registries[registry_count]);
-    //registry_count++;
+    clib_package_t *package = clib_package_load_local_manifest(0);
+    registries_t registries = registry_manager_init_registries(package->registries, secrets);
+    registry_manager_fetch_registries(registries);
 
     // TODO, implement caching for the new registries.
     /*
@@ -210,10 +198,12 @@ int main(int argc, char *argv[]) {
     debug(&debugger, "found %zu packages", pkgs->len);
      */
 
-    for (int i = 0; i < registry_count; i++) {
-        printf("SEARCH: packages from %s\n", wiki_registry_get_url(registries[i]));
+    registry_iterator_t it = registry_iterator_new(registries);
+    wiki_registry_ptr_t registry = NULL;
+    while ((registry = registry_iterator_next(it))) {
+        printf("SEARCH: packages from %s\n", wiki_registry_get_url(registry));
         wiki_package_ptr_t pkg;
-        wiki_registry_iterator_t it = wiki_registry_iterator_new(registries[i]);
+        wiki_registry_iterator_t it = wiki_registry_iterator_new(registry);
 
         JSON_Array *json_list = NULL;
         JSON_Value *json_list_root = NULL;
@@ -233,7 +223,7 @@ int main(int argc, char *argv[]) {
                     display_package(pkg, fg_color_highlight, fg_color_text);
                 }
             } else {
-                debug(&debugger, "skipped package %s", wiki_package_get_repo(pkg));
+                debug(&debugger, "skipped package %s", wiki_package_get_id(pkg));
             }
         }
 
@@ -246,7 +236,6 @@ int main(int argc, char *argv[]) {
         }
 
         wiki_registry_iterator_destroy(it);
-        wiki_registry_free(registries[i]);
     }
     command_free(&program);
     return 0;

src/clib-update.c

@@ -230,7 +230,7 @@ static int install_package(const char *slug) {
   }
 
   if (!pkg) {
-    pkg = clib_package_new_from_slug(slug, opts.verbose);
+    pkg = clib_package_new_from_slug_and_url(slug, opts.verbose);
   }
 
   if (NULL == pkg)