Fix resolving requirements recursively (#396)

Fix resolving requirements recursively

Author: dschep

lib/pip.js

@@ -338,6 +338,28 @@ function dockerPathForWin(path) {
     return path;
   }
 }
+
+/**
+ * get requirements from requirements.txt
+ * @param {string} source
+ * @return {string[]}
+ */
+function getRequirements(source) {
+  const requirements = fse
+    .readFileSync(source, { encoding: 'utf-8' })
+    .replace(/\\\n/g, ' ')
+    .split(/\r?\n/);
+
+  return requirements.reduce((acc, req) => {
+    req = req.trim();
+    if (!req.startsWith('-r')) {
+      return [...acc, req];
+    }
+    source = path.join(path.dirname(source), req.replace(/^-r\s+/, ''));
+    return [...acc, ...getRequirements(source)];
+  }, []);
+}
+
 /** create a filtered requirements.txt without anything from noDeploy
  *  then remove all comments and empty lines, and sort the list which
  *  assist with matching the static cache.  The sorting will skip any
@@ -351,10 +373,7 @@ function dockerPathForWin(path) {
  */
 function filterRequirementsFile(source, target, options) {
   const noDeploy = new Set(options.noDeploy || []);
-  const requirements = fse
-    .readFileSync(source, { encoding: 'utf-8' })
-    .replace(/\\\n/g, ' ')
-    .split(/\r?\n/);
+  const requirements = getRequirements(source);
   var prepend = [];
   const filteredRequirements = requirements.filter(req => {
     req = req.trim();

test.js

@@ -170,6 +170,21 @@ test('py3.6 can package flask with hashes', t => {
   t.end();
 });
 
+test('py3.6 can package flask with nested', t => {
+  process.chdir('tests/base');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls([
+    `--pythonBin=${getPythonBin(3)}`,
+    '--fileName=requirements-w-nested.txt',
+    'package'
+  ]);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged');
+  t.end();
+});
+
 test('py3.6 can package flask with zip option', t => {
   process.chdir('tests/base');
   const path = npm(['pack', '../..']);
@@ -1471,6 +1486,10 @@ test('py3.6 can package only requirements of module', t => {
     zipfiles_hello.includes(`pyaml${sep}__init__.py`),
     'pyaml is packaged in function hello1'
   );
+  t.true(
+    zipfiles_hello.includes(`boto3${sep}__init__.py`),
+    'boto3 is packaged in function hello1'
+  );
   t.false(
     zipfiles_hello.includes(`flask${sep}__init__.py`),
     'flask is NOT packaged in function hello1'
@@ -1491,6 +1510,10 @@ test('py3.6 can package only requirements of module', t => {
     zipfiles_hello2.includes(`pyaml${sep}__init__.py`),
     'pyaml is NOT packaged in function hello2'
   );
+  t.false(
+    zipfiles_hello2.includes(`boto3${sep}__init__.py`),
+    'boto3 is NOT packaged in function hello2'
+  );
   t.true(
     zipfiles_hello2.includes(`flask${sep}__init__.py`),
     'flask is packaged in function hello2'

tests/base/requirements-common.txt

@@ -0,0 +1 @@
+boto3

tests/base/requirements-w-nested.txt

@@ -0,0 +1,3 @@
+flask
+bottle
+-r requirements-common.txt

tests/individually/module1/requirements.txt

@@ -1 +1,2 @@
+-r ../requirements-common.txt
 pyaml

tests/individually/requirements-common.txt

@@ -0,0 +1 @@
+boto3