diff --git a/Dockerfile b/Dockerfile index ac347bf..1b0991e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,18 +16,24 @@ RUN go mod download COPY ./cmd ./cmd COPY ./pkg ./pkg WORKDIR /go/metahub/cmd/boltdb -# static build +# static build of boltdb backend +ENV CGO_ENABLED=0 GOOS=linux +RUN go build -a -ldflags '-extldflags "-static"' . +WORKDIR /go/metahub/cmd/static +# static build of static backend ENV CGO_ENABLED=0 GOOS=linux RUN go build -a -ldflags '-extldflags "-static"' . EXPOSE 8080 + # Go binary serves the ui web content FROM scratch +ARG MH_BACKEND=boltdb ENV PORT=80 COPY --from=go /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=ui /go/metahub/static /srv/html/static COPY --from=ui /go/metahub/templates/gen/index.html /srv/html/ -COPY --from=go /go/metahub/cmd/boltdb/boltdb /usr/bin/ +COPY --from=go /go/metahub/cmd/${MH_BACKEND}/${MH_BACKEND} /usr/bin/metahub VOLUME /data/ WORKDIR /data/ -ENTRYPOINT ["/usr/bin/boltdb"] +ENTRYPOINT ["/usr/bin/metahub"] diff --git a/cmd/static/main.go b/cmd/static/main.go new file mode 100644 index 0000000..ea2dd99 --- /dev/null +++ b/cmd/static/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + "log" + "metahub/pkg/daemon" + registry "metahub/pkg/registry/http/client" + "metahub/pkg/storage/static" + "net/http" + "os" + + "metahub/cmd" +) + +// Log allows to make http interactions visible +func Log(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL) + handler.ServeHTTP(w, r) + }) +} + +func main() { + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + + storageService := static.NewService() + registryService := registry.NewService() + daemonService := daemon.NewService(storageService, registryService) + + router := cmd.RegisterRoutes(daemonService) + + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), Log(router))) +} diff --git a/go.mod b/go.mod index 155637e..71922b0 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/gorilla/mux v1.7.1 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 // indirect + //github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/qnib/image-spec v2.0.0-feature+incompatible github.com/sirupsen/logrus v1.4.1 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 ) diff --git a/go.sum b/go.sum index 4692dbf..e85b27c 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/qnib/image-spec v2.0.0-feature+incompatible h1:Tkei126H6GlnhaTj1ipFEGRWnDxQ4C1KeaJC/5O8DOE= +github.com/qnib/image-spec v2.0.0-feature+incompatible/go.mod h1:KDRMn7EhYdZqS2LTyV+C1uWDYMd4kJ2Bw9iBu3U1ftk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= diff --git a/pkg/storage/boltdb/dummies.go b/pkg/storage/boltdb/dummies.go deleted file mode 100644 index 98e0eca..0000000 --- a/pkg/storage/boltdb/dummies.go +++ /dev/null @@ -1,41 +0,0 @@ -package boltdb - -import "metahub/pkg/storage" - -// Consts for protoype -const ( - user = "qnib" - accountName = user -) - -// Dummy MachineTypes -var ( - mType1 = storage.MachineType{ - ID : 1, - DisplayName : "type1", - Features : []string{"cpu:broadwell"}, - Login : user+"-type1", - Password : user+"-type1", - } - mType2 = storage.MachineType{ - ID : 2, - DisplayName : "type2", - Features : []string{"cpu:skylake"}, - Login : user+"-type2", - Password : user+"-type2", - } - mType3 = storage.MachineType{ - ID : 3, - DisplayName : "type3", - Features : []string{"cpu:coffelake"}, - Login : user+"-type3", - Password : user+"-type3", - } - mType4 = storage.MachineType{ - ID : 4, - DisplayName : "type4", - Features : []string{"cpu:broadwell","nvcap:5.2"}, - Login : user+"-type4", - Password : user+"-type4", - } -) diff --git a/pkg/storage/boltdb/machinetype_service.go b/pkg/storage/boltdb/machinetype_service.go index cfbf438..86c9389 100644 --- a/pkg/storage/boltdb/machinetype_service.go +++ b/pkg/storage/boltdb/machinetype_service.go @@ -6,12 +6,12 @@ import ( "fmt" "log" "metahub/pkg/storage" + "github.com/boltdb/bolt" - "os" ) type machineTypeService struct { - ctx context.Context + ctx context.Context } func formatLogin(accountName string, login string) { @@ -19,21 +19,6 @@ func formatLogin(accountName string, login string) { } func (s *machineTypeService) GetByID(accountName string, id int64) (mt *storage.MachineType, err error) { - log.Printf("GetByID(%s, %d)\n", accountName, id) - if _, b := os.LookupEnv("STATIC_MACHINES");b { - log.Println("Environment STATIC_MACHINES is set: Hardcoded types are served") - switch id { - case 1: - mt = &mType1 - case 2: - mt = &mType2 - case 3: - mt = &mType3 - case 4: - mt = &mType4 - } - return mt, nil - } var foundID bool db.View(func(tx *bolt.Tx) error { // Assume bucket exists and has keys @@ -53,37 +38,17 @@ func (s *machineTypeService) GetByID(accountName string, id int64) (mt *storage. } return err }) - if !foundID { - err = fmt.Errorf("Could not find MachineType with ID: %i", id) + if !foundID { + err = fmt.Errorf("Could not find MachineType with ID: %d", id) } return mt, err } func (s *machineTypeService) GetByUsername(username string) (mt *storage.MachineType, err error) { - log.Printf("GetByUsername(%s)\n", username) - if _, b := os.LookupEnv("STATIC_MACHINES");b { - log.Println("Environment STATIC_MACHINES is set: Serve static machine type") - switch username { - case user+"-type1": - return &mType1, nil - case user+"-type2": - return &mType2, nil - case user+"-type3": - return &mType3, nil - case user+"-type4": - return &mType4, nil - default: - panic(fmt.Errorf("Could not find username: %s", username)) - } - } return mt, nil } func (s *machineTypeService) Add(accountName string, mt *storage.MachineType) (err error) { - if _, b := os.LookupEnv("STATIC_MACHINES");b { - log.Println("Environment STATIC_MACHINES is set: Skip Add()") - return err - } dbSync.Lock() defer dbSync.Unlock() err = db.Update(func(tx *bolt.Tx) error { @@ -106,15 +71,6 @@ func (s *machineTypeService) Delete(accountName string, id int64) error { func (s *machineTypeService) List(accountName string) ([]storage.MachineType, error) { result := []storage.MachineType{} - if _, b := os.LookupEnv("STATIC_MACHINES");b { - log.Println("Environment STATIC_MACHINES is set: Serve static machine type") - return []storage.MachineType{ - mType1, - mType2, - mType3, - mType4, - },nil - } db.View(func(tx *bolt.Tx) error { // Assume bucket exists and has keys b := tx.Bucket([]byte("TYPES")) @@ -122,7 +78,7 @@ func (s *machineTypeService) List(accountName string) ([]storage.MachineType, er c := b.Cursor() var mt storage.MachineType for k, v := c.First(); k != nil; k, v = c.Next() { - if err := json.Unmarshal(v, &mt); err != nil { + if err := json.Unmarshal(v, &mt); err != nil { panic(err) } result = append(result, mt) @@ -133,10 +89,6 @@ func (s *machineTypeService) List(accountName string) ([]storage.MachineType, er } func (s *machineTypeService) Update(accountName string, mt storage.MachineType) (err error) { - if _, b := os.LookupEnv("STATIC_MACHINES");b { - log.Println("Environment STATIC_MACHINES is set: Serve static machine type") - return - } dbSync.Lock() defer dbSync.Unlock() err = db.Update(func(tx *bolt.Tx) error { diff --git a/pkg/storage/machinetype.go b/pkg/storage/machinetype.go index 51cf943..84fd9be 100644 --- a/pkg/storage/machinetype.go +++ b/pkg/storage/machinetype.go @@ -14,8 +14,9 @@ type MachineType struct { Password string `json:"password"` } +// ToBytes creates a Byte array func (mtm MachineType) ToBytes() []byte { reqBodyBytes := new(bytes.Buffer) json.NewEncoder(reqBodyBytes).Encode(mtm) return reqBodyBytes.Bytes() -} \ No newline at end of file +} diff --git a/pkg/storage/manifestlist.go b/pkg/storage/manifestlist.go new file mode 100644 index 0000000..9fdcc9a --- /dev/null +++ b/pkg/storage/manifestlist.go @@ -0,0 +1,76 @@ +package storage + +import ( + "bytes" + "encoding/json" + "fmt" + + v1 "github.com/qnib/image-spec/specs-go/v2" +) + +// repoName represents the full path without the registry component. +// https://docs.docker.com/registry/spec/api/ +/****** +Classically, repository names have always been two path components where each path component +is less than 30 characters. The V2 registry API does not enforce this. +The rules for a repository name are as follows: +1. A repository name is broken up into path components. A component of a +repository name must be at least one lowercase, alpha-numeric characters, optionally separated by periods, +dashes or underscores. More strictly, it must match the regular expression [a-z0-9]+(?:[._-][a-z0-9]+)*. +2. If a repository name has two or more path components, they must be separated by a forward slash (“/”). +3. The total length of a repository name, including slashes, must be less than 256 characters. +*/ + +// ManifestList describes the metahub-internal definitions of a ManifestList +type ManifestList struct { + ID int64 `json:"id"` + RepoName string `json:"repo"` + TagName string `json:"tag"` + Manifests []Manifest `json:"manifests,omitempty"` + // TODO: Do we need user/password to safeguard ManifestLists - might be just safeguarded vai Manifest +} + +// ToBytes returns a bytearray +func (mtm ManifestList) ToBytes() []byte { + reqBodyBytes := new(bytes.Buffer) + json.NewEncoder(reqBodyBytes).Encode(mtm) + return reqBodyBytes.Bytes() +} + +// NewManifestList generates a new Manifest list +func NewManifestList(repo, tag string, mfs ...Manifest) (ml ManifestList, err error) { + ml.RepoName = repo + ml.TagName = tag + ml.Manifests = mfs + return +} + +// Manifest describes a single Image (aka Manifest) +type Manifest struct { + ID int64 `json:"id"` + RepoName string `json:"image"` + TagName string `json:"tag"` + Platform v1.Platform `json:"features,omitempty"` +} + +// ToBytes returns a bytearray +func (mtm Manifest) ToBytes() []byte { + reqBodyBytes := new(bytes.Buffer) + json.NewEncoder(reqBodyBytes).Encode(mtm) + return reqBodyBytes.Bytes() +} + +// NewManifest generates a new Manifest objects checking for correctness +func NewManifest(repo, tag, os, arch string, features ...string) (mf Manifest, err error) { + mf.RepoName = repo + mf.TagName = tag + if arch != "amd64" && os != "linux" { + return mf, fmt.Errorf("Let's first concentrate on 'linux' and 'amd64' - keep it simple") + } + mf.Platform = v1.Platform{ + Architecture: arch, + OS: os, + Features: features, + } + return +} diff --git a/pkg/storage/manifestlist_service.go b/pkg/storage/manifestlist_service.go new file mode 100644 index 0000000..11d238b --- /dev/null +++ b/pkg/storage/manifestlist_service.go @@ -0,0 +1,11 @@ +package storage + +// ManifestListService provides an internal storage for ManifestLists defined within MetaHub +// to complement backend ManifestList or backends without ManifestList implementation +type ManifestListService interface { + HasManifestList(repoName string) (result bool, err error) + Add(ml *ManifestList) (err error) + Delete(repoName, tagName string, id int64) error + List() ([]*ManifestList, error) + Update(ml *ManifestList) error +} diff --git a/pkg/storage/static/accesstoken_model.go b/pkg/storage/static/accesstoken_model.go new file mode 100644 index 0000000..0e9cc44 --- /dev/null +++ b/pkg/storage/static/accesstoken_model.go @@ -0,0 +1,12 @@ +package static + +import ( + "time" +) + +var accessTokenEntityKind = "access_token" + +type accessToken struct { + AccountName string `datastore:"account,noindex"` + Expiry time.Time `datastore:"expiry,omitempty"` +} diff --git a/pkg/storage/static/accesstoken_service.go b/pkg/storage/static/accesstoken_service.go new file mode 100644 index 0000000..a99a9be --- /dev/null +++ b/pkg/storage/static/accesstoken_service.go @@ -0,0 +1,23 @@ +package static + +import ( + "context" + "metahub/pkg/storage" + "time" +) + +type accessTokenService struct { + ctx context.Context +} + +func (s *accessTokenService) Get(token string) (*storage.AccessToken, error) { + //TODO: check at.Expiry? + return &storage.AccessToken{ + AccountName: token, + Expiry: time.Time{}, + }, nil +} + +func (s *accessTokenService) Put(token string, at storage.AccessToken) error { + return nil +} diff --git a/pkg/storage/static/account_model.go b/pkg/storage/static/account_model.go new file mode 100644 index 0000000..cc4083b --- /dev/null +++ b/pkg/storage/static/account_model.go @@ -0,0 +1,7 @@ +package static + +var accountEntityKind = "account" + +type account struct { + DisplayName string `datastore:"name,noindex"` +} diff --git a/pkg/storage/static/account_service.go b/pkg/storage/static/account_service.go new file mode 100644 index 0000000..afd539e --- /dev/null +++ b/pkg/storage/static/account_service.go @@ -0,0 +1,20 @@ +package static + +import ( + "context" + "metahub/pkg/storage" +) + +type accountService struct { + ctx context.Context +} + +func (s *accountService) Upsert(name string, a storage.Account) error { + return nil +} + +func (s *accountService) Get(name string) (*storage.Account, error) { + return &storage.Account{ + DisplayName: name, + }, nil +} diff --git a/pkg/storage/static/dummies.go b/pkg/storage/static/dummies.go new file mode 100644 index 0000000..fcf0a21 --- /dev/null +++ b/pkg/storage/static/dummies.go @@ -0,0 +1,56 @@ +package static + +import ( + "metahub/pkg/storage" +) + +// Consts for protoype +const ( + user = "qnib" + accountName = user +) + +// Dummy MachineTypes +var ( + mType1 = storage.MachineType{ + ID: 1, + DisplayName: "type1", + Features: []string{"cpu:broadwell"}, + Login: user + "-type1", + Password: user + "-type1", + } + mType2 = storage.MachineType{ + ID: 2, + DisplayName: "type2", + Features: []string{"cpu:skylake"}, + Login: user + "-type2", + Password: user + "-type2", + } + mType3 = storage.MachineType{ + ID: 3, + DisplayName: "type3", + Features: []string{"cpu:coffelake"}, + Login: user + "-type3", + Password: user + "-type3", + } + mType4 = storage.MachineType{ + ID: 4, + DisplayName: "type4", + Features: []string{"cpu:broadwell", "nvcap:5.2"}, + Login: user + "-type4", + Password: user + "-type4", + } + // Dummy Manifests + mfUbuntu, _ = storage.NewManifest("ubuntu", "latest", "linux", "amd64") + mfQFeatGeneric, _ = storage.NewManifest("qnib/plain-featuretest", "generic", "linux", "amd64") + mfQFeatBroadwell, _ = storage.NewManifest("qnib/plain-featuretest", "cpu-broadwell", "linux", "amd64", "cpu:broadwell") + mfQFeatSkylake, _ = storage.NewManifest("qnib/plain-featuretest", "cpu-skylake", "linux", "amd64", "cpu:skylake") + // Dummy ManifestLists + mlQBench, _ = storage.NewManifestList("qnib/bench", "test", mfQFeatGeneric, mfQFeatBroadwell, mfQFeatSkylake) +) + +func getMachineTypes() []storage.MachineType { + return []storage.MachineType{ + mType1, mType2, mType3, mType4, + } +} diff --git a/pkg/storage/static/machinetype_model.go b/pkg/storage/static/machinetype_model.go new file mode 100644 index 0000000..95da954 --- /dev/null +++ b/pkg/storage/static/machinetype_model.go @@ -0,0 +1,10 @@ +package static + +var machineTypeEntityKind = "MachineType" + +type machineTypeModel struct { + DisplayName string `datastore:"name,noindex"` + Features []string `datastore:"features,noindex"` + Login string `datastore:"login"` + Password string `datastore:"password,noindex"` +} diff --git a/pkg/storage/static/machinetype_service.go b/pkg/storage/static/machinetype_service.go new file mode 100644 index 0000000..f83a79f --- /dev/null +++ b/pkg/storage/static/machinetype_service.go @@ -0,0 +1,67 @@ +package static + +import ( + "context" + "fmt" + "log" + "metahub/pkg/storage" +) + +type machineTypeService struct { + ctx context.Context +} + +func formatLogin(accountName string, login string) { + +} + +func (s *machineTypeService) GetByID(accountName string, id int64) (mt *storage.MachineType, err error) { + log.Printf("GetByID(%s, %d)\n", accountName, id) + switch id { + case 1: + mt = &mType1 + case 2: + mt = &mType2 + case 3: + mt = &mType3 + case 4: + mt = &mType4 + } + return mt, nil + +} + +func (s *machineTypeService) GetByUsername(username string) (mt *storage.MachineType, err error) { + log.Printf("GetByUsername(%s)\n", username) + switch username { + case user + "-type1": + return &mType1, nil + case user + "-type2": + return &mType2, nil + case user + "-type3": + return &mType3, nil + case user + "-type4": + return &mType4, nil + default: + panic(fmt.Errorf("Could not find username: %s", username)) + } +} + +func (s *machineTypeService) Add(accountName string, mt *storage.MachineType) (err error) { + log.Println("Environment STATIC_MACHINES is set: Skip Add()") + return err +} + +func (s *machineTypeService) Delete(accountName string, id int64) error { + log.Println("Environment STATIC_MACHINES is set: Skip Delete()") + return nil +} + +func (s *machineTypeService) List(accountName string) ([]storage.MachineType, error) { + return mTypes, nil +} + +func (s *machineTypeService) Update(accountName string, mt storage.MachineType) (err error) { + log.Println("Environment STATIC_MACHINES is set: Serve static machine type") + return +} diff --git a/pkg/storage/static/manifestlist_model.go b/pkg/storage/static/manifestlist_model.go new file mode 100644 index 0000000..2d6cb24 --- /dev/null +++ b/pkg/storage/static/manifestlist_model.go @@ -0,0 +1,3 @@ +package static + +var manifestListEntityKind = "ManifestListType" diff --git a/pkg/storage/static/manifestlist_service.go b/pkg/storage/static/manifestlist_service.go new file mode 100644 index 0000000..d5dab3f --- /dev/null +++ b/pkg/storage/static/manifestlist_service.go @@ -0,0 +1,37 @@ +package static + +import ( + "context" + "log" + + "metahub/pkg/storage" +) + +type manifestListService struct { + ctx context.Context +} + +// Add would normaly add a new ManifestList - but in static that is not possible.. +func (s *manifestListService) Add(ml *storage.ManifestList) (err error) { + log.Println("Static backend does not support Add()") + return +} + +// Delete would normaly remove a new ManifestList - but in static that is not possible.. +func (s *manifestListService) Delete(repoName, tagName string, id int64) (err error) { + log.Println("Static backend does not support delete") + return +} + +// Update would normaly overwrite a new ManifestList - but in static that is not possible.. +func (s *manifestListService) Update(ml *storage.ManifestList) (err error) { + log.Println("Static backend does not support updates") + return +} + +// List is the only function that does something... :) +func (s *manifestListService) List() ([]*storage.ManifestList, error) { + return []*storage.ManifestList{ + &mlQBench, + }, nil +} diff --git a/pkg/storage/static/service.go b/pkg/storage/static/service.go new file mode 100644 index 0000000..92826d1 --- /dev/null +++ b/pkg/storage/static/service.go @@ -0,0 +1,50 @@ +package static + +import ( + "context" + "fmt" + "metahub/pkg/storage" + "sync" + + "github.com/boltdb/bolt" +) + +var db *bolt.DB +var dbSync sync.Mutex +var mTypes []storage.MachineType + +func init() { + mTypes = getMachineTypes() + setupData() +} + +// NewService returns a new storage.Service for boltdb +func NewService() storage.Service { + return &service{} +} + +type service struct { +} + +func (s *service) MachineTypeService(ctx context.Context) (storage.MachineTypeService, error) { + return &machineTypeService{ + ctx: ctx, + }, nil +} + +func (s *service) AccessTokenService(ctx context.Context) (storage.AccessTokenService, error) { + return &accessTokenService{ + ctx: ctx, + }, nil +} + +func (s *service) AccountService(ctx context.Context) (storage.AccountService, error) { + return &accountService{ + ctx: ctx, + }, nil +} + +func setupData() error { + fmt.Println("Data Setup Done") + return nil +} diff --git a/ui/src/App.vue b/ui/src/App.vue index a828fe0..2c9fd7e 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -26,6 +26,15 @@ + + + dashboard + + + Manifest Lists + + + diff --git a/ui/src/router.js b/ui/src/router.js index e310459..384f6f4 100644 --- a/ui/src/router.js +++ b/ui/src/router.js @@ -6,6 +6,8 @@ Vue.use(Router) import Welcome from "./views/Welcome"; import MachineTypes from "./views/MachineTypes"; import MachineType from "./views/MachineType" +import ManifestLists from "./views/ManifestLists"; +import ManifestList from "./views/ManifestList" const router = new Router({ mode: 'hash', @@ -44,6 +46,35 @@ const router = new Router({ requiresAuth: true, } }, + { + name: "manifest-lists", + path: '/manifestlists', + components: { + default: MachineTypes, + }, + meta: { + title: "Manifest Lists", + requiresAuth: false, + } + }, + { + name: "edit-manifest-list", + path: '/manifestlists/edit/:id', + component: MachineType, + meta: { + title: "Manifest List details", + requiresAuth: false, + } + }, + { + name: "new-manifest-list", + path: '/manifestlists/new', + component: MachineType, + meta: { + title: "Manifest List details", + requiresAuth: false, + } + }, ] }) router.beforeEach((to, from, next) => { diff --git a/ui/src/views/ManifestList.vue b/ui/src/views/ManifestList.vue new file mode 100644 index 0000000..03ca5aa --- /dev/null +++ b/ui/src/views/ManifestList.vue @@ -0,0 +1,183 @@ + + + + + diff --git a/ui/src/views/ManifestLists.vue b/ui/src/views/ManifestLists.vue new file mode 100644 index 0000000..1a2a2f2 --- /dev/null +++ b/ui/src/views/ManifestLists.vue @@ -0,0 +1,75 @@ + + + + +