Merge pull request #82 from vsoch/v2.4.1

V2.4.1

Author: vsoch

.travis/before_install

@@ -1,10 +1,13 @@
 #!/bin/bash
 
-apt-get update && apt-get install -y wget git build-essential squashfs-tools \
-                                          libtool \
-                                          autotools-dev \
-                                          automake \
-                                          autoconf
+sudo apt-get update && sudo apt-get install -y wget git build-essential squashfs-tools \
+                                                    libtool \
+                                                    autotools-dev \
+                                                    automake \
+                                                    autoconf \
+                                                    uuid-dev \
+                                                    libssl-dev
+
 
 sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers
 

CHANGELOG.md

@@ -13,8 +13,16 @@ and changes prior to that are (unfortunately) done retrospectively. Critical ite
  - migration guidance (how to convert images?)
  - changed behaviour (recipe sections work differently)
 
-
 ## [vxx](https://github.com/singularityware/singularity-python/tree/development) (development)
 
+
+## [v2.4.1](https://github.com/singularityware/singularity-python/releases/tag/v2.4.1) (v2.4.1)
+
+**changed behavior**
+ - *apps extraction* files limited to those in bin and lib, as the response to the server was too large.
+ - **updated tests** to handle secure build
+
+## [v2.4](https://github.com/singularityware/singularity-python/releases/tag/v2.4) (v2.4)
+
 **changed defaults**
  - *sregistry client*: to support use of squashfs images and singularity 2.4, the default upload is not compressed, assuming squashfs, and the default download is not decompressed. To still compress an image add the `--compress` flag on push, and the `--decompress` flag on pull.

singularity/analysis/apps.py

@@ -79,7 +79,9 @@ def extract_apps(image_path, app_names, S=None, verbose=True):
         except:
             pass
         base = '/scif/apps/%s' %app_name
-        metadata['files'] = [x.path for x in members if base in x.path]
+        libs = [x.path for x in members if "%s/lib" %base in x.path]
+        bins = [x.path for x in members if "%s/bin" %base in x.path]
+        metadata['files'] = libs + bins
         apps[app_name] = metadata
 
     return apps

singularity/analysis/classify.py

@@ -128,7 +128,12 @@ def estimate_os(container=None,image_package=None,return_top=True):
         comparison = compare_packages(packages_set1=[image_package])['files.txt'].transpose()
 
     comparison.columns = ['SCORE']
-    most_similar = comparison['SCORE'].idxmax()
+    try:
+        most_similar = comparison['SCORE'].idxmax()
+    except:
+        most_similar = comparison['SCORE'].values.argmax()
+        most_similar = comparison.index.tolist()[most_similar]
+
     print("Most similar OS found to be ", most_similar)    
     if return_top == True:
         return most_similar

singularity/build/main.py

@@ -162,6 +162,7 @@ def run_build(build_dir,params,verbose=True, compress_image=False):
         diff = get_diff(image_package=image_package)
 
         # Inspect to get labels and other metadata
+
         cli = Singularity(debug=params['debug'])
         inspect = cli.inspect(image_path=image)
 
@@ -179,7 +180,8 @@ def run_build(build_dir,params,verbose=True, compress_image=False):
         extensions = extension_counts(diff=diff)
 
         os_sims = estimate_os(image_package=image_package,return_top=False)
-        most_similar = os_sims['SCORE'].idxmax()
+        most_similar = os_sims['SCORE'].values.argmax()
+        most_similar = os_sims['SCORE'].index.tolist()[most_similar]
 
         metrics = {'build_time_seconds':final_time,
                    'singularity_version':singularity_version,
@@ -238,6 +240,9 @@ def send_build_data(build_dir, data, secret,
 
     if response_url is not None:
         finish = requests.post(response_url,data=data, headers=headers)
+        bot.debug("RECEIVE POST TO SINGULARITY HUB ---------------------")
+        bot.debug(finish.status_code)
+        bot.debug(finish.reason)
     else:
         bot.warning("response_url set to None, skipping sending of build.")
 
@@ -275,5 +280,8 @@ def send_build_close(params,response_url):
 
     headers = {'Authorization': signature }
 
-    # Send it back!
-    return requests.post(response_url,data=response, headers=headers)
+    finish = requests.post(response_url,data=response, headers=headers)
+    bot.debug("FINISH POST TO SINGULARITY HUB ---------------------")
+    bot.debug(finish.status_code)
+    bot.debug(finish.reason)
+    return finish

singularity/build/scripts/singularity-prepare-instance.sh

@@ -18,6 +18,8 @@ sudo apt-get -y install git \
                    autoconf \
                    debootstrap \
                    yum \
+                   uuid-dev \
+                   libssl-dev \
                    python3-pip
 
 
@@ -29,9 +31,9 @@ sudo pip3 install --upgrade google &&
 sudo pip3 install oauth2client==3.0.0
 
 # Install Singularity from Github
-cd /tmp && git clone -b feature-squashbuild-secbuild https://github.com/cclerget/singularity.git &&
+cd /tmp && git clone -b feature-squashbuild-secbuild-2.4.1 https://github.com/cclerget/singularity.git &&
 cd /tmp/singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install && sudo make secbuildimg
 
 # Singularity python development
-cd /tmp && git clone -b development https://www.github.com/vsoch/singularity-python.git &&
+cd /tmp && git clone -b v2.4.1 https://www.github.com/vsoch/singularity-python.git &&
 cd /tmp/singularity-python && sudo python3 setup.py install

singularity/cli/__init__.py

@@ -319,9 +319,9 @@ def pull(self,image_path,pull_folder='',
             if size is not None:
                 cmd = cmd + ["--size",size]
             if image_name is None:
-                image_name = "%s" %image_path.replace("docker://","").replace("/","-")
-            final_image = "%s%s.img" %(pull_folder,image_name)
-            cmd = cmd + ["--name", image_name]
+                image_name = image_path.replace("docker://","").replace("/","-")
+            final_image = "%s%s.simg" %(pull_folder,image_name)
+            cmd = cmd + ["--name", final_image]
 
         cmd.append(image_path)
         bot.debug(' '.join(cmd))

singularity/version.py

@@ -22,7 +22,7 @@
 SOFTWARE.
 '''
 
-__version__ = "1.2.1"
+__version__ = "2.4.1"
 AUTHOR = 'Vanessa Sochat'
 AUTHOR_EMAIL = 'vsochat@stanford.edu'
 NAME = 'singularity'