Add support for custom registries and repositories, read secrets from

clib_secrets.json

misc: Cleanup, add headers.

Author: nouwaarom

Makefile

@@ -12,20 +12,22 @@ RM      = rm -f
 MKDIR   = mkdir -p
 
 SRC  = $(wildcard src/*.c)
-COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c)
-ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h test/package/*.c test/cache/*.c)
+COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*)
+ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h src/repository/*.h src/repository/*.c test/package/*.c test/cache/*.c)
 SDEPS = $(wildcard deps/*/*.c)
 ODEPS = $(SDEPS:.c=.o)
 DEPS = $(filter-out $(ODEPS), $(SDEPS))
 OBJS = $(DEPS:.c=.o)
 
 export CC
 
+CFLAGS  += -std=gnu17 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -g -Wall -Werror=return-type -Wno-unused-function $(shell curl-config --cflags)
+
 ifdef STATIC
-	CFLAGS  += -DCURL_STATICLIB -std=c99 -Ideps -Wall -Werror=return-type -Wno-unused-function -U__STRICT_ANSI__ $(shell deps/curl/bin/curl-config --cflags)
+	CFLAGS  += -DCURL_STATICLIB $(shell deps/curl/bin/curl-config --cflags)
 	LDFLAGS += -static $(shell deps/curl/bin/curl-config --static-libs)
 else
-	CFLAGS  += -std=gnu17 -Ideps -g -Wall -Werror=return-type -Wno-unused-function $(shell curl-config --cflags)
+	CFLAGS  += $(shell curl-config --cflags)
 	LDFLAGS += $(shell curl-config --libs)
 endif
 

clib.json

@@ -44,7 +44,7 @@
     "stephenmathieson/gumbo-get-element-by-id.c": "*",
     "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*",
     "clibs/list": "*",
-    "jwerle/url.h": "0.0.*"
+    "jwerle/url.h": "0.1.*"
   },
   "development": {
     "stephenmathieson/describe.h": "2.0.1"

deps/http-get/http-get.c

@@ -97,7 +97,7 @@ static size_t http_get_file_cb(void *ptr, size_t size, size_t nmemb, void *strea
  * Request `url` and save to `file`
  */
 
-int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
+int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** headers, int header_count) {
   CURL *req = curl_easy_init();
   if (!req) return -1;
 
@@ -108,6 +108,15 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
     curl_easy_setopt(req, CURLOPT_SHARE, share);
   }
 
+  if (header_count > 0) {
+    struct curl_slist *chunk = NULL;
+    for (int i = 0; i < header_count; i++) {
+      chunk = curl_slist_append(chunk, headers[i]);
+    }
+    /* set our custom set of headers */
+    curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk);
+  }
+
   curl_easy_setopt(req, CURLOPT_URL, url);
   curl_easy_setopt(req, CURLOPT_HTTPGET, 1);
   curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1);
@@ -125,7 +134,7 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) {
 }
 
 int http_get_file(const char *url, const char *file) {
-  return http_get_file_shared(url, file, NULL);
+  return http_get_file_shared(url, file, NULL, NULL, 0);
 }
 
 /**

deps/http-get/http-get.h

@@ -25,7 +25,7 @@ http_get_response_t *http_get(const char *url, const char** headers, int header_
 http_get_response_t *http_get_shared(const char *url, void *, const char** headers, int header_count);
 
 int http_get_file(const char *, const char *);
-int http_get_file_shared(const char *, const char *, void *);
+int http_get_file_shared(const char *, const char *, void *, const char** headers, int header_count);
 
 void http_get_free(http_get_response_t *);
 

deps/url/url.h

@@ -51,6 +51,7 @@
  * http://en.wikipedia.org/wiki/URI_scheme
  */
 
+#ifdef URL_H_IMPLEMENTATION
 char *URL_SCHEMES[] = {
         // official IANA registered schemes
         "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw",
@@ -72,6 +73,7 @@ char *URL_SCHEMES[] = {
         // unofficial schemes
         "javascript", "jdbc", "doi"
 };
+#endif
 
 
 /**
@@ -102,7 +104,7 @@ typedef struct url_data {
  */
 
 url_data_t *
-url_parse (char *url);
+url_parse (const char *url);
 
 char *
 url_get_protocol (char *url);
@@ -151,7 +153,7 @@ url_data_inspect (url_data_t *data);
 
 
 // implementation
-
+#ifdef URL_H_IMPLEMENTATION
 
 // non C99 standard functions
 #if _POSIX_C_SOURCE < 200809L
@@ -220,7 +222,7 @@ get_part (char *url, const char *format, int l) {
 }
 
 url_data_t *
-url_parse (char *url) {
+url_parse (const char *url) {
     url_data_t *data = malloc(sizeof(url_data_t));
     if (!data) return NULL;
 
@@ -590,5 +592,6 @@ url_free (url_data_t *data) {
     free(data);
 }
 
+#endif
 
 #endif

src/clib-build.c

@@ -199,7 +199,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) {
 #ifdef DEBUG
     package = clib_package_new_from_slug(dir, 1);
 #else
-    package = clib_package_new_from_slug(dir, 0);
+    package = clib_package_new_from_slug_and_url(dir, "FIXME", 0);
 #endif
   }
 
@@ -330,7 +330,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) {
       char *dep_dir = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0);
       if (opts.dir && dependency && dependency->name) {
         dep_dir = path_join(opts.dir, dependency->name);
       }
@@ -401,7 +401,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) {
       char *slug = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0);
       char *dep_dir = path_join(opts.dir, dependency->name);
 
       free(slug);

src/clib-configure.c

@@ -192,7 +192,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) {
 #ifdef DEBUG
     package = clib_package_new_from_slug(dir, 1);
 #else
-    package = clib_package_new_from_slug(dir, 0);
+    package = clib_package_new_from_slug_and_url(dir, "FIXME", 0);
 #endif
   }
 
@@ -288,7 +288,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) {
       char *slug = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0);
       char *dep_dir = path_join(opts.dir, dependency->name);
 
       free(slug);
@@ -359,7 +359,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) {
       char *slug = 0;
       asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version);
 
-      clib_package_t *dependency = clib_package_new_from_slug(slug, 0);
+      clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0);
       char *dep_dir = path_join(opts.dir, dependency->name);
 
       free(slug);

src/clib-install.c

@@ -17,9 +17,12 @@
 #include "parson/parson.h"
 #include "str-replace/str-replace.h"
 #include "version.h"
+#include <clib-secrets.h>
 #include <curl/curl.h>
 #include <libgen.h>
 #include <limits.h>
+#include <registry-manager.h>
+#include <repository.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -251,21 +254,6 @@ static int install_package(const char *slug) {
   long path_max = 4096;
 #endif
 
-  if (!root_package) {
-    const char *name = NULL;
-    char *json = NULL;
-    unsigned int i = 0;
-
-    do {
-      name = manifest_names[i];
-      json = fs_read(name);
-    } while (NULL != manifest_names[++i] && !json);
-
-    if (json) {
-      root_package = clib_package_new(json, opts.verbose);
-    }
-  }
-
   if ('.' == slug[0]) {
     if (1 == strlen(slug) || ('/' == slug[1] && 2 == strlen(slug))) {
       char dir[path_max];
@@ -291,10 +279,21 @@ static int install_package(const char *slug) {
     }
   }
 
-  if (!pkg) {
-    pkg = clib_package_new_from_slug(slug, opts.verbose);
+  // Read local config files.
+  clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json");
+  repository_init(secrets); // The repository requires the secrets for authentication.
+  clib_package_t *package = clib_package_load_local_manifest(0);
+
+  registries_t registries = registry_manager_init_registries(package->registries, secrets);
+  registry_manager_fetch_registries(registries);
+  registry_package_ptr_t package_info = registry_manger_find_package(registries, slug);
+  if (!package_info) {
+    debug(&debugger, "Package %s not found in any registry.", slug);
+    return -1;
   }
 
+
+  pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), opts.verbose);
   if (NULL == pkg)
     return -1;
 
@@ -437,6 +436,21 @@ int main(int argc, char *argv[]) {
 
   clib_package_set_opts(package_opts);
 
+  if (!root_package) {
+    const char *name = NULL;
+    char *json = NULL;
+    unsigned int i = 0;
+
+    do {
+      name = manifest_names[i];
+      json = fs_read(name);
+    } while (NULL != manifest_names[++i] && !json);
+
+    if (json) {
+      root_package = clib_package_new(json, opts.verbose);
+    }
+  }
+
   int code = 0 == program.argc ? install_local_packages()
                                : install_packages(program.argc, program.argv);