Add support for "-e hg+URI" style dependencies

initial work was done by @seppeljordan in #101

then I (@garbas):
- rebased on current master
- added support for pinning repositories to branch/revision
- used https://bitbucket.org/tarek/flake8 as example repository

fixes #100
fixes #101

Author: garbas

.travis.yml

@@ -9,6 +9,7 @@ env:
   - EXAMPLE=empy
   - EXAMPLE=ldap
   - EXAMPLE=lektor
+  - EXAMPLE=mercurial
   - EXAMPLE=rss2email
   - EXAMPLE=scipy
   - EXAMPLE=tornado
@@ -20,6 +21,9 @@ matrix:
     - os: osx
       env: EXAMPLE=scipy
 before_install:
+  - nix-env -iA nixpkgs.cacert -p result-cacert
+  - export SSL_CERT_FILE=$PWD/result-cacert/etc/ssl/certs/ca-bundle.crt
+  - sudo mkdir -p /etc/ssl/certs/ && sudo rm -f /etc/ssl/certs/ca-certificates.crt && sudo ln -s $PWD/result-cacert/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
   - echo 'binary-caches = https://cache.nixos.org/ https://travis.garbas.si/pypi2nix/' | sudo tee -a /etc/nix/nix.conf > /dev/null
   - openssl aes-256-cbc -K $encrypted_0cfcc1944c73_key -iv $encrypted_0cfcc1944c73_iv -in deploy_rsa.enc -out deploy_rsa -d
   - eval "$(ssh-agent -s)"

default.nix

@@ -44,6 +44,7 @@ in stdenv.mkDerivation rec {
   patchPhase = ''
     sed -i -e "s|default='nix-shell',|default='${nix.out}/bin/nix-shell',|" $out/pkgs/pypi2nix/cli.py
     sed -i -e "s|nix-prefetch-git|${nix-prefetch-scripts}/bin/nix-prefetch-git|" $out/pkgs/pypi2nix/stage2.py
+    sed -i -e "s|nix-prefetch-hg|${nix-prefetch-scripts}/bin/nix-prefetch-hg|" $out/pkgs/pypi2nix/stage2.py
   '';
 
   commonPhase = ''

examples/Makefile

@@ -10,13 +10,24 @@ else
 LEKTOR_BUILD_INPUTS="libffi openssl unzip"
 endif
 
+.PHONY: \
+	awscli_and_requests \
+	empy \
+	ldap \
+	lektor \
+	mercurial \
+	rss2email \
+	scipy \
+	sentry \
+	tornado \
+	vulnix
 
-# commented out sentry since it doesn't work on travis
 all: \
 	awscli_and_requests \
 	empy \
 	ldap \
 	lektor \
+	mercurial \
 	rss2email \
 	scipy \
 	sentry \
@@ -26,8 +37,9 @@ all: \
 clear: \
 	awscli_and_requests-clear \
 	empy-clear \
-	ldap_clear \
+	ldap-clear \
 	lektor-clear \
+	mercurial-clear \
 	pypi2nix-clear \
 	rss2email-clear \
 	scipy-clear \
@@ -227,6 +239,10 @@ tornado: tornado.nix
 	@nix-build tornado.nix -o tornado -A interpreter --show-trace
 	@./tornado/bin/python -c 'import tornado'
 	@nix-shell tornado.nix -A interpreter --run "python -c 'import tornado'"
+	@if [ `cat tornado_generated.nix | grep "69253c820df473407c562a227d0ba36df25018ab" | wc -l` != "1" ]; then \
+			echo "ERROR: Revision '69253c820df473407c562a227d0ba36df25018ab' not found in tornado_generated.nix!"; \
+			exit 123; \
+		fi
 
 tornado.nix: tornado.txt $(PYPI2NIX)
 	@echo "generating tornado.nix expressions ..."
@@ -270,3 +286,31 @@ ldap-clear:
 	@rm -f ldap.nix
 	@rm -f ldap_generated.nix
 	@rm -f ldap_frozen.txt
+
+
+mercurial: mercurial.nix
+	@echo "building and testing mercurial..."
+	@nix-build mercurial.nix -o mercurial -A interpreter --show-trace -j 1
+	@./mercurial/bin/python -c 'import flake8'
+	@nix-shell mercurial.nix -A interpreter --run "python -c 'import flake8'"
+	@if [ `cat mercurial_generated.nix | grep "a209fb6" | wc -l` != "1" ]; then \
+			echo "ERROR: Revision 'a209fb6' not found in tornado_generated.nix!"; \
+			exit 123; \
+		fi
+
+mercurial.nix: mercurial.txt $(PYPI2NIX)
+	@echo "generating mercurial.nix expressions ..."
+	@$(PYPI2NIX) -v --basename mercurial -r mercurial.txt -I $(NIX_PATH) -V "2.7" -E "mercurial"
+
+
+mercurial.txt: mercurial-clear
+	@echo "-e hg+https://bitbucket.org/tarek/flake8@a209fb6#egg=flake8" > mercurial.txt
+	@echo "pytest-runner" >> mercurial.txt
+	@echo "setuptools-scm" >> mercurial.txt
+
+mercurial-clear:
+	@rm -f mercurial
+	@rm -f mercurial.txt
+	@rm -f mercurial.nix
+	@rm -f mercurial_generated.nix
+	@rm -f mercurial_frozen.txt

examples/mercurial_override.nix

@@ -0,0 +1,16 @@
+{ pkgs, python }:
+
+self: super: {
+
+   "pyflakes" = python.overrideDerivation super."pyflakes" (old: {
+      buildInputs = [ self."pytest-runner" ];
+   });
+
+   "mccabe" = python.overrideDerivation super."mccabe" (old: {
+      buildInputs = [ self."pytest-runner" ];
+   });
+
+   "pytest-runner" = python.overrideDerivation super."pytest-runner" (old: {
+      buildInputs = [ self."setuptools-scm" ];
+   });
+}

src/pypi2nix/cli.py

@@ -192,7 +192,8 @@ def handle_requirements_file(project_dir, requirements_file):
         with open(requirements_file) as f1:
             with open(new_requirements_file, "w+") as f2:
                 for requirements_line in f1.readlines():
-                    if requirements_line.startswith("-e git+"):
+                    if requirements_line.startswith("-e git+") or \
+                           requirements_line.startswith("-e hg+"):
                         pass
                     elif requirements_line.startswith("-e"):
                         requirements_line = "-e %s" % (

src/pypi2nix/stage2.py

@@ -258,8 +258,6 @@ def find_release(wheel_cache_dir, wheel, wheel_data):
             break
 
     if not wheel_release:
-        import pdb
-        pdb.set_trace()
         raise click.ClickException(
             "Unable to find release for package {name} of version "
             "{version}".format(**wheel))
@@ -285,17 +283,20 @@ def find_release(wheel_cache_dir, wheel, wheel_data):
     return release
 
 
-def process_wheel(wheel_cache_dir, wheel, sources, verbose, index=INDEX_URL,
-                  chunk_size=2048):
+def process_wheel(wheel_cache_dir, wheel, sources, repo_types, verbose,
+                  index=INDEX_URL, chunk_size=2048):
     """
     """
 
+    repo_type = repo_types.get(wheel['name'], None)
     if wheel['name'] in sources:
         release = dict()
         release['url'] = sources[wheel['name']]
         release['hash_type'] = 'sha256'
 
-        if release['url'].startswith('http://') or release['url'].startswith('https://'):
+        if repo_type is None and \
+               (release['url'].startswith('http://') or \
+                release['url'].startswith('https://')):
 
             release['fetch_type'] = 'fetchurl'
 
@@ -309,16 +310,18 @@ def process_wheel(wheel_cache_dir, wheel, sources, verbose, index=INDEX_URL,
                     hash = hashlib.sha256(fd.read())
 
             release['hash_value'] = hash.hexdigest()
-        elif release['url'].startswith('git'):
+
+        elif repo_type == 'git':
             revision = ''
             if release['url'].startswith('git+'):
                 release['url'] = release['url'][4:]
             if '@' in release['url']:
                 release['url'], revision = release['url'].split('@')
+
             release['fetch_type'] = 'fetchgit'
             command = 'nix-prefetch-git {url} {revision}'.format(
                 url=release['url'],
-                revision=revision
+                revision=revision,
             )
             return_code, output = cmd(command, verbose != 0)
             if return_code != 0:
@@ -342,6 +345,43 @@ def process_wheel(wheel_cache_dir, wheel, sources, verbose, index=INDEX_URL,
                     output=output
                 ))
 
+        elif repo_type == 'hg':
+            revision = ''
+            if release['url'].startswith('hg+'):
+                release['url'] = release['url'][3:]
+            if '@' in release['url']:
+                release['url'], revision = release['url'].split('@')
+
+            release['fetch_type'] = 'fetchhg'
+            command = 'nix-prefetch-hg {url} {revision}'.format(
+                url=release['url'],
+                revision=revision,
+            )
+            return_code, output = cmd(command, verbose != 0)
+            if return_code != 0:
+                raise click.ClickException("URL {url} for package {name} is not valid.".format(
+                    url=release['url'],
+                    name=wheel['name']
+                ))
+            HASH_PREFIX = 'hash is '
+            REV_PREFIX = 'hg revision is '
+            for output_line in output.split('\n'):
+                print(output_line)
+                output_line = output_line.strip()
+                if output_line.startswith(HASH_PREFIX):
+                    release['hash_value'] = output_line[len(HASH_PREFIX):].strip()
+                elif output_line.startswith(REV_PREFIX):
+                    release['rev'] = output_line[len(REV_PREFIX):].strip()
+
+            if release.get('hash_value', None) is None:
+                raise click.ClickException('Could not determine the hash from ouput:\n{output}'.format(
+                    output=output
+                ))
+            if release.get('rev', None) is None:
+                raise click.ClickException('Could not determine the revision from ouput:\n{output}'.format(
+                    output=output
+                ))
+
     else:
         url = "{}/{}/json".format(index, wheel['name'])
         r = requests.get(url, timeout=None)
@@ -366,6 +406,7 @@ def main(verbose, wheels, requirements_files, wheel_cache_dir, index=INDEX_URL):
 
     # get url's from requirements_files
     sources = dict()
+    repo_types = dict()
     for requirements_file in requirements_files:
         with open(requirements_file) as f:
             lines = f.readlines()
@@ -374,11 +415,15 @@ def main(verbose, wheels, requirements_files, wheel_cache_dir, index=INDEX_URL):
                 if line.startswith('-e '):
                     line = line[3:]
                 if line.startswith('http://') or line.startswith('https://') or \
-                   line.startswith('git+'):
+                   line.startswith('git+') or line.startswith('hg+'):
                     try:
                         url, egg = line.split('#')
                         name = egg.split('egg=')[1]
-                        sources[name] = url
+                        sources[name] = url.replace('-e git+','').replace('-e hg+','')
+                        if line.startswith('git+'):
+                            repo_types[name] = 'git'
+                        elif line.startswith('hg+'):
+                            repo_types[name] = 'hg'
                     except:
                         raise click.ClickException(
                             "Requirement starting with http:// or https:// "
@@ -416,7 +461,8 @@ def main(verbose, wheels, requirements_files, wheel_cache_dir, index=INDEX_URL):
                 click.echo("--------------------------------------------------------------------------")
 
             metadata.append(
-                process_wheel(wheel_cache_dir, wheel_metadata, sources, verbose, index))
+                process_wheel(wheel_cache_dir, wheel_metadata, sources,
+                              repo_types, verbose, index))
     except Exception as e:
         if verbose == 0:
             click.echo(output)

src/pypi2nix/stage3.py

@@ -152,6 +152,13 @@ def main(packages_metadata,
                 hash_value=item['hash_value'],
                 rev=item['rev']
             )
+        elif fetch_type == 'fetchhg':
+            fetch_expression = 'pkgs.fetchhg { url = "%(url)s"; %(hash_type)s = "%(hash_value)s"; rev = "%(rev)s"; }' % dict(
+                url=item['url'],
+                hash_type=item['hash_type'],
+                hash_value=item['hash_value'],
+                rev=item['rev']
+            )
         else:
             fetch_expression='pkgs.fetchurl { url = "%(url)s"; %(hash_type)s = "%(hash_value)s"; }' % dict(
                 url=item['url'],

src/pypi2nix/utils.py

@@ -41,7 +41,6 @@ def cmd(command, verbose=False):
     if isinstance(command, str):
         command = shlex.split(command)
 
-    output = '|-> ' + ' '.join(command)
     if verbose:
         click.echo('|-> ' + ' '.join(command))
 
@@ -57,12 +56,11 @@ def cmd(command, verbose=False):
         if line == '' and p.poll() is not None:
             break
         if line != '':
-            output += '    ' + line
             if verbose:
                 click.echo('    ' + line.rstrip('\n'))
             out.append(line)
 
-    return p.returncode, output
+    return p.returncode, '\n'.join(out)
 
 
 def create_command_options(options):