Merge pull request #68 from vsoch/development

Development update for singularity 2.3

Author: vsoch

singularity/build/google.py

@@ -27,11 +27,6 @@
 
 '''
 
-from singularity.build.api import (
-    api_get, 
-    api_put
-)
-
 from singularity.build.utils import sniff_extension
 
 from singularity.build.main import (
@@ -363,14 +358,13 @@ def get_build_metadata(key):
     '''
     headers = {"Metadata-Flavor":"Google"}
     url = "http://metadata.google.internal/computeMetadata/v1/instance/attributes/%s" %(key)        
-    response = api_get(url=url,headers=headers)
-
+    response = requests.get(url=url,headers=headers)
     # Successful query returns the result
     if response.status_code == 200:
         return response.text
     else:
-        bot.logger.error("Error retrieving metadata %s, returned response %s", key,
-                                                                               response.status_code)
+        bot.logger.debug("Metadata %s not present, returned response %s", key,
+                                                                          response.status_code)
     return None
 
 

singularity/build/scripts/singularity-build-latest.sh

@@ -5,6 +5,7 @@ sudo apt-get -y install git \
                    build-essential \
                    libtool \
                    autotools-dev \
+                   debootstrap \
                    automake \
                    autoconf \
                    python3-pip >> /tmp/.install-log

singularity/cli.py

@@ -44,6 +44,7 @@
 import tempfile
 import shutil
 import json
+import sys
 import os
 import re
 
@@ -101,13 +102,41 @@ def help(self,command=None,stdout=True):
             return help
 
 
-    def println(self,output):
-        '''print will print the output, given that quiet is not True
+    def println(self,output,quiet=False):
+        '''print will print the output, given that quiet is not True. This
+        function also serves to convert output in bytes to utf-8
         '''
-        if self.quiet is False:
-            if isinstance(output,bytes):
-                output = output.decode('utf-8')
+        if isinstance(output,bytes):
+            output = output.decode('utf-8')
+
+        if self.quiet is False and quiet is False:
             print(output)
+        return output
+
+
+    def bootstrap(self,image_path,spec_path):
+        '''create will bootstrap an image using a spec
+        :param image_path: full path to image
+        :param spec_path: full path to the spec file (Singularity)
+        ''' 
+        if self.debug == True:
+            cmd = ['singularity','--debug','bootstrap',image_path,spec_path]
+        else:
+            cmd = ['singularity','bootstrap',image_path,spec_path]
+        output = self.run_command(cmd,sudo=True)
+        output = self.println(output)     
+        return image_path
+
+
+    def compress(self,image_path):
+        '''compress will (properly) compress an image'''
+        if os.path.exists(image_path):
+            compressed_image = "%s.gz" %image_path
+            os.system('gzip -c -9 %s > %s' %(image_path,compressed_image))
+            return compressed_image
+        else:
+            bot.logger.error("Cannot find image %s",image_path)
+        return None
 
 
     def create(self,image_path,size=None,sudo=False):
@@ -124,24 +153,21 @@ def create(self,image_path,size=None,sudo=False):
         else:
             cmd = ['singularity','create','--size',str(size),image_path]
         output = self.run_command(cmd,sudo=sudo)
-        self.println(output)        
+        output = self.println(output)        
         if os.path.exists(image_path):
             return image_path
         return None
 
 
-    def bootstrap(self,image_path,spec_path):
-        '''create will bootstrap an image using a spec
-        :param image_path: full path to image
-        :param spec_path: full path to the spec file (Singularity)
-        ''' 
-        if self.debug == True:
-            cmd = ['singularity','--debug','bootstrap',image_path,spec_path]
+
+    def decompress(self,image_path):
+        '''decompress will (properly) decompress an image'''
+        if os.path.exists(image_path):
+            extracted_file = image_path.replace('.gz','')
+            os.system('gzip -d -f %s' %image_path)
+            return extracted_file
         else:
-            cmd = ['singularity','bootstrap',image_path,spec_path]
-        output = self.run_command(cmd,sudo=True)
-        self.println(output)        
-        return image_path
+            bot.logger.error("Cannot find image %s",image_path)
 
 
     def execute(self,image_path,command,writable=False,contain=False):
@@ -205,42 +231,78 @@ def importcmd(self,image_path,input_source):
         '''
         cmd = ['singularity','import',image_path,input_source]
         output = self.run_command(cmd,sudo=False)
-        self.println(output)        
+        output = self.println(output)        
         return image_path
 
 
-    def pull(self,image_path,pull_folder=None,name_by=None):
+    def inspect(self,image_path,json=True,labels=True,
+                runscript=True,test=True,deffile=True,
+                environment=True,quiet=False):
+        '''inspect will show labels, defile, runscript, and tests for an image
+        :param image_path: path of image to inspect
+        :param json: print json instead of raw text (default True)
+        :param labels: show labels (default True):
+        :param runscript: show runscript (default True)
+        :param test: show test file (default True)
+        :param environment: show environment (default True)
+        '''
+        cmd = ['singularity','--quiet','inspect']
+        options = locals()
+        acceptable = ['environment','json','deffile','labels','runscript','test']
+        for key,option in options.items():
+            if key in acceptable:
+                if option is True:
+                    cmd.append('--%s' %(key))
+
+        cmd.append(image_path)
+        output = self.run_command(cmd)
+        output = self.println(output,quiet=quiet)    
+        return output
+
+
+    def pull(self,image_path,pull_folder=None,
+                             name_by_hash=False,
+                             name_by_commit=False,
+                             image_name=None):
         '''pull will pull a singularity hub image
         :param image_path: full path to image
         :param name_by: can be one of commit or hash, default is by image name
         ''' 
-        if name_by is None:
-            name_by = "name"
-        name_by = name_by.lower()
+        if image_name is not None:
+            name_by_hash=False
+            name_by_commit=False
 
         if pull_folder is not None:
             os.environ['SINGULARITY_PULLFOLDER'] = pull_folder
 
-        if not image_path.startswith('shub://'):
-            bot.logger.error("pull is only valid for the shub://uri, %s is invalid.",image_name)
+        if not image_path.startswith('shub://') and not image_path.startswith('docker://'):
+            bot.logger.error("pull is only valid for docker and shub, %s is invalid.",image_name)
             sys.exit(1)           
 
         if self.debug == True:
             cmd = ['singularity','--debug','pull']
         else:
             cmd = ['singularity','pull']
 
-        if name_by in ['name','commit','hash']:
-            bot.logger.debug("user specified naming pulled image by %s",name_by)
-            name_by = "--%s " %name_by
-            cmd.append(name_by)
+        if image_path.startswith('shub://'):
+            if image_name is not None:
+                bot.logger.debug("user specified naming pulled image %s",image_name)
+                cmd = cmd +["--name",image_name]
+            elif name_by_commit is True:
+                bot.logger.debug("user specified naming by commit.")
+                cmd.append("--commit")
+            elif name_by_hash is True:
+                bot.logger.debug("user specified naming by hash.")
+                cmd.append("--hash")
+ 
+        elif image_path.startswith('docker://'):
+            if image_name is not None:
+                bot.logger.debug("user specified name of image as %s",image_name)
+                cmd = cmd + ["--name",image_name]
 
         cmd.append(image_path)
-
         output = self.run_command(cmd)
-        self.println(output)        
-        if isinstance(output,bytes):
-            output = output.decode('utf-8')
+        output = self.println(output,quiet=True)        
         return output.split("Container is at:")[-1].strip('\n').strip()
 
 
@@ -266,8 +328,7 @@ def run(self,image_path,args=None,writable=False,contain=False):
             cmd = cmd + args
 
         result = self.run_command(cmd,sudo=sudo)
-        if isinstance(result,bytes):
-            result = result.decode('utf-8')
+        result = self.println(result,quiet=True)
         result = result.strip('\n')
         try:
             result = json.loads(result)
@@ -281,8 +342,7 @@ def get_labels(self,image_path):
         '''
         cmd = ['singularity','exec',image_path,'cat','/.singularity/labels.json']
         labels = self.run_command(cmd)
-        if isinstance(labels,bytes):
-            labels = labels.decode('utf-8')
+        labels = self.println(labels,quiet=True)
         if len(labels) > 0:
             return json.loads(labels)
         return labels
@@ -302,13 +362,11 @@ def get_args(self,image_path):
         return args
 
 
-    def add_flags(self,cmd,writable,contain):
+    def add_flags(self,cmd,writable=False,contain=False):
         '''check_args is a general function for adding flags to a command list
         :param writable: adds --writable
         :param contain: adds --contain
-        '''
-
-        # Does the user want to make the container writeable?
+        ''' 
         if writable == True:
             cmd.append('--writable')       
 

singularity/hub/auth.py

@@ -33,6 +33,23 @@
 import sys
 
 
+def basic_auth_header(username, password):
+    '''basic_auth_header will return a base64 encoded header object to
+    generate a token
+    :param username: the username
+    :param password: the password
+    '''
+    s = "%s:%s" % (username, password)
+    if sys.version_info[0] >= 3:
+        s = bytes(s, 'utf-8')
+        credentials = base64.b64encode(s).decode('utf-8')
+    else:
+        credentials = base64.b64encode(s)
+    auth = {"Authorization": "Basic %s" % credentials}
+    return auth
+
+
+
 def get_headers(token=None):
     '''get_headers will return a simple default header for a json
     post. This function will be adopted as needed.

singularity/package.py

@@ -178,13 +178,6 @@ def package(image_path,spec_path=None,output_folder=None,runscript=True,
     if spec_path is not None:
         singularity_spec = "".join(read_file(spec_path))
         to_package['Singularity'] = singularity_spec
-
-    # Package the image with an sha1, identical standard, as VERSION
-    if old_version == False:
-        hashes = get_image_hashes(image_path)
-        to_package["HASHES"] = hashes
-        to_package["VERSION"] = hashes['IDENTICAL']
-    else:
         to_package["VERSION"] = get_image_file_hash(image_path)
 
 

singularity/tests/test_client.py

@@ -47,8 +47,6 @@ def setUp(self):
         self.pwd = get_installdir()
         self.cli = Singularity()
         self.tmpdir = tempfile.mkdtemp()
-        self.image1 = "%s/tests/data/busybox-2016-02-16.img" %(self.pwd)
-        self.image2 = "%s/tests/data/cirros-2016-01-04.img" %(self.pwd)
 
     def tearDown(self):
         shutil.rmtree(self.tmpdir)
@@ -75,37 +73,67 @@ def test_import(self):
         self.cli.importcmd(container,'docker://ubuntu')
         result = test_container(container)
         self.assertEqual(result['return_code'],0)
+        os.remove(container)
 
-    def test_run(self):
-        print("Testing client.run command")
+    def test_exec(self):
+        print('Testing client.execute command')
+        container = create_container(do_import=True) 
+        result = self.cli.execute(container,'ls /')
+        print(result)
+        os.remove(container)
+        #if isinstance(result,bytes):
+        #    result = result.decode('utf-8')
+        #self.assertTrue(len(result)>0)
+
+
+    def test_inspect(self):
+        print("Testing client.inspect command")
         container = create_container(do_import=True)
-        result = self.cli.run(container)
-        self.assertEqual(result,'')
+        result = self.cli.inspect(container,quiet=True)
+        labels = json.loads(result)
+        self.assertTrue('data' in labels)     
+        os.remove(container)
+
 
     def test_exec(self):
         print('Testing client.execute command')
         container = create_container(do_import=True) 
         result = self.cli.execute(container,'ls /')
         print(result)
+        os.remove(container)
+
         #if isinstance(result,bytes):
         #    result = result.decode('utf-8')
         #self.assertTrue(len(result)>0)
 
 
+
     def test_pull(self):
         print("Testing client.pull command")
 
         print("Case 1: Testing naming pull by image name")
         image = self.cli.pull("shub://vsoch/singularity-images")
+        self.assertTrue(os.path.exists(image))
         print(image)
+        os.remove(image)
 
         print("Case 2: Testing naming pull by image commit")
-        image = self.cli.pull("shub://vsoch/singularity-images",name_by="commit")
+        image = self.cli.pull("shub://vsoch/singularity-images",name_by_commit=True)
+        self.assertTrue(os.path.exists(image))
         print(image)
+        os.remove(image)
 
         print("Case 3: Testing naming pull by image hash")
-        image = self.cli.pull("shub://vsoch/singularity-images",name_by="hash")
+        image = self.cli.pull("shub://vsoch/singularity-images",name_by_hash=True)
+        self.assertTrue(os.path.exists(image))
+        print(image)
+        os.remove(image)
+
+        print("Case 3: Testing docker pull")
+        image = self.cli.pull("docker://ubuntu:14.04")
         print(image)
+        self.assertTrue(os.path.exists(image))
+        os.remove(image)
 
 
     def test_get_image(self):
@@ -125,9 +153,10 @@ def create_container(container=None,do_import=False):
     tmpdir = tempfile.mkdtemp()
     if container is None:
         container = "%s/container.img" %(tmpdir)
+    container = cli.create(container)
     if do_import is True:
         cli.importcmd(container,'docker://ubuntu')
-    return cli.create(container)
+    return container
 
 
 if __name__ == '__main__':

singularity/version.py

@@ -22,7 +22,7 @@
 SOFTWARE.
 '''
 
-__version__ = "1.1.8"
+__version__ = "1.1.9"
 AUTHOR = 'Vanessa Sochat'
 AUTHOR_EMAIL = 'vsochat@stanford.edu'
 NAME = 'singularity'