diff --git a/lisp/core/analyze.el b/lisp/core/analyze.el
index 882c4b55..13a715f1 100644
--- a/lisp/core/analyze.el
+++ b/lisp/core/analyze.el
@@ -29,14 +29,20 @@
 (defvar eask-analyze--warnings nil)
 (defvar eask-analyze--errors nil)
 
+;; Warning flag
+(defvar eask-analyze--warning-p nil)
+;; Error flag
+(defvar eask-analyze--error-p nil)
+
 (defun eask-analyze--pretty-json (json)
   "Return pretty JSON."
   (with-temp-buffer (insert json) (json-pretty-print-buffer) (buffer-string)))
 
 (defun eask-analyze--load-buffer ()
   "Return the current file loading session."
-  (car (cl-remove-if-not
-        (lambda (elm) (string-prefix-p " *load*-" (buffer-name elm))) (buffer-list))))
+  (car (cl-remove-if-not (lambda (elm)
+                           (string-prefix-p " *load*-" (buffer-name elm)))
+                         (buffer-list))))
 
 (defun eask-analyze--write-json-format (level msg)
   "Prepare log for JSON format.
@@ -87,6 +93,10 @@ information."
 
 Argument LEVEL and MSG are data from the debug log signal."
   (unless (string= " *temp*" (buffer-name))  ; avoid error from `package-file' directive
+    (when (eq 'error level)
+      (setq eask-analyze--error-p t))
+    (when (eq 'warn level)
+      (setq eask-analyze--warning-p t))
     (with-current-buffer (or (eask-analyze--load-buffer) (buffer-name))
       (funcall
        (cond ((eask-json-p) #'eask-analyze--write-json-format)
@@ -100,19 +110,22 @@ Argument LEVEL and MSG are data from the debug log signal."
     (dolist (file files)
       (eask--silent-error
         (eask--save-load-eask-file file
+            (push file checked-files)
+            ;; also count files with errors in the total count
             (push file checked-files))))
 
     ;; Print result
     (eask-msg "")
-    (cond ((and (eask-json-p)  ; JSON format
-                (or eask-analyze--warnings eask-analyze--errors))
-           (setq content
-                 (eask-analyze--pretty-json (json-encode
-                                             `((warnings . ,eask-analyze--warnings)
-                                               (errors   . ,eask-analyze--errors)))))
+    (cond ((eask-json-p)  ; JSON format
+           ;; Fill content with result.
+           (when (or eask-analyze--warnings eask-analyze--errors)
+             (setq content
+                   (eask-analyze--pretty-json (json-encode
+                                               `((warnings . ,eask-analyze--warnings)
+                                                 (errors   . ,eask-analyze--errors))))))
            ;; XXX: When printing the result, no color allow.
            (eask--with-no-color
-             (eask-msg content)))
+             (eask-msg (or content "{}"))))
           (eask-analyze--log  ; Plain text
            (setq content
                  (with-temp-buffer
@@ -121,11 +134,11 @@ Argument LEVEL and MSG are data from the debug log signal."
                    (buffer-string)))
            ;; XXX: When printing the result, no color allow.
            (eask--with-no-color
-             (mapc #'eask-msg (reverse eask-analyze--log))))
-          (t
-           (eask-info "(Checked %s file%s)"
-                      (length checked-files)
-                      (eask--sinr checked-files "" "s"))))
+             (mapc #'eask-msg (reverse eask-analyze--log)))))
+
+    (eask-info "(Checked %s file%s)"
+               (length checked-files)
+               (eask--sinr checked-files "" "s"))
 
     ;; Output file
     (when (and content (eask-output))
@@ -148,7 +161,11 @@ Argument LEVEL and MSG are data from the debug log signal."
   (cond
    ;; Files found, do the action!
    (files
-    (eask-analyze--file files))
+    (eask-analyze--file files)
+    (when (or eask-analyze--error-p
+              ;; strict flag turns warnings into errors
+              (and eask-analyze--warning-p (eask-strict-p)))
+      (eask--exit 'failure)))
    ;; Pattern defined, but no file found!
    (patterns
     (eask-info "(No files match wildcard: %s)"
diff --git a/test/jest/__snapshots__/analyze.test.js.snap b/test/jest/__snapshots__/analyze.test.js.snap
index 71298295..65506abe 100644
--- a/test/jest/__snapshots__/analyze.test.js.snap
+++ b/test/jest/__snapshots__/analyze.test.js.snap
@@ -1,6 +1,309 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`analyze in ./dsl matches snapshot 1`] = `
+exports[`analyze in ./analyze/dsl handles json option 1`] = `
+{
+  "stderr": "
+{
+  "warnings": [
+    {
+      "range": {
+        "start": {
+          "line": 25,
+          "col": 0,
+          "pos": 853
+        },
+        "end": {
+          "line": 25,
+          "col": 35,
+          "pos": 888
+        }
+      },
+      "filename": "~/Eask",
+      "message": "💡 Pkg-file seems to be missing \`check-pkg.el'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 20,
+          "col": 0,
+          "pos": 709
+        },
+        "end": {
+          "line": 20,
+          "col": 17,
+          "pos": 726
+        }
+      },
+      "filename": "~/Eask",
+      "message": "💡 Warning regarding duplicate license name, GPLv3"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 17,
+          "col": 0,
+          "pos": 613
+        },
+        "end": {
+          "line": 17,
+          "col": 15,
+          "pos": 628
+        }
+      },
+      "filename": "~/Eask",
+      "message": "💡 Warning regarding duplicate author name, name"
+    }
+  ],
+  "errors": [
+    {
+      "range": {
+        "start": {
+          "line": 46,
+          "col": 0,
+          "pos": 1696
+        },
+        "end": {
+          "line": 50,
+          "col": 2,
+          "pos": 1908
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Define dependencies with the same name \`f' with different version"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 46,
+          "col": 0,
+          "pos": 1696
+        },
+        "end": {
+          "line": 50,
+          "col": 2,
+          "pos": 1908
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Define dependencies with the same name \`f'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 44,
+          "col": 0,
+          "pos": 1596
+        },
+        "end": {
+          "line": 44,
+          "col": 27,
+          "pos": 1623
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Define dependencies with the same name \`dash' with different version"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 43,
+          "col": 0,
+          "pos": 1576
+        },
+        "end": {
+          "line": 43,
+          "col": 19,
+          "pos": 1595
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Define dependencies with the same name \`dash'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 40,
+          "col": 0,
+          "pos": 1474
+        },
+        "end": {
+          "line": 40,
+          "col": 20,
+          "pos": 1494
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Define dependencies with the same name \`emacs'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 37,
+          "col": 0,
+          "pos": 1304
+        },
+        "end": {
+          "line": 37,
+          "col": 15,
+          "pos": 1319
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Unknown package archive \`local'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 37,
+          "col": 0,
+          "pos": 1304
+        },
+        "end": {
+          "line": 37,
+          "col": 15,
+          "pos": 1319
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Invalid archive name \`local'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 35,
+          "col": 0,
+          "pos": 1229
+        },
+        "end": {
+          "line": 35,
+          "col": 24,
+          "pos": 1253
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Unknown package archive \`magic-archive'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 33,
+          "col": 0,
+          "pos": 1214
+        },
+        "end": {
+          "line": 33,
+          "col": 13,
+          "pos": 1227
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Multiple definition of source \`gnu'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 30,
+          "col": 0,
+          "pos": 1073
+        },
+        "end": {
+          "line": 30,
+          "col": 61,
+          "pos": 1134
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Run-script with the same key name is not allowed: \`test\`"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 26,
+          "col": 0,
+          "pos": 953
+        },
+        "end": {
+          "line": 26,
+          "col": 35,
+          "pos": 988
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Multiple definition of \`package-descriptor'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 23,
+          "col": 0,
+          "pos": 822
+        },
+        "end": {
+          "line": 23,
+          "col": 29,
+          "pos": 851
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Multiple definition of \`package-file'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 14,
+          "col": 0,
+          "pos": 517
+        },
+        "end": {
+          "line": 14,
+          "col": 16,
+          "pos": 533
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Multiple definition of \`keywords'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 12,
+          "col": 0,
+          "pos": 383
+        },
+        "end": {
+          "line": 12,
+          "col": 55,
+          "pos": 438
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Multiple definition of \`website-url'"
+    },
+    {
+      "range": {
+        "start": {
+          "line": 9,
+          "col": 0,
+          "pos": 290
+        },
+        "end": {
+          "line": 9,
+          "col": 18,
+          "pos": 308
+        }
+      },
+      "filename": "~/Eask",
+      "message": "✗ Multiple definition of \`package'"
+    }
+  ]
+}
+(Checked 1 file)
+",
+  "stdout": "",
+}
+`;
+
+exports[`analyze in ./analyze/dsl matches snapshot 1`] = `
 {
   "stderr": "
 ~/Eask:9:18 Error: ✗ Multiple definition of \`package'
@@ -21,6 +324,7 @@ exports[`analyze in ./dsl matches snapshot 1`] = `
 ~/Eask:44:27 Error: ✗ Define dependencies with the same name \`dash' with different version
 ~/Eask:50:2 Error: ✗ Define dependencies with the same name \`f'
 ~/Eask:50:2 Error: ✗ Define dependencies with the same name \`f' with different version
+(Checked 1 file)
 ",
   "stdout": "",
 }
diff --git a/test/jest/analyze.test.js b/test/jest/analyze.test.js
index cef00035..1054be49 100644
--- a/test/jest/analyze.test.js
+++ b/test/jest/analyze.test.js
@@ -1,32 +1,78 @@
 const { TestContext } = require("./helpers");
 
+/**
+ * Clean output and attempt to parse as JSON.
+ * Throws if failing.
+ * @param {string} s
+ * @returns {object} the parsed JSON.
+ */
+function tryJSON(s) {
+  // The main use case of analyze --json is flycheck/flymake
+  // both of which accept mixed JSON/text output.
+  // So the test should filter non-JSON output in a similar way
+  // See flycheck--json-parser for reference
+
+  // This is a rough approximation of the greedy JSON parsing
+  // that flycheck uses
+  // It will yield a false negative if:
+  // - a non-JSON output line begins with a space
+  // - there is more that one JSON object in the output
+  const lines = s.split("\n");
+  const json = lines.filter((x) => x.match("^[{}\\[\\] ]"));
+  if (json.length == 0) {
+    return {};
+  } else {
+    const result = JSON.parse(json.join("\n"));
+    if (typeof result == "string") {
+      // this case is a single string prefixed with whitespace
+      return {};
+    }
+    return result;
+  }
+}
+
 describe("analyze", () => {
-  describe("in ./dsl", () => {
-    const ctx = new TestContext("./test/jest/dsl");
+  describe("in ./analyze/dsl", () => {
+    const ctx = new TestContext("./test/jest/analyze/dsl");
 
     it("handles plain text", async () => {
-      await ctx.runEask("analyze");
-      await ctx.runEask("analyze Eask");
+      await expect(ctx.runEask("analyze")).rejects.toThrow();
+      await expect(ctx.runEask("analyze Eask")).rejects.toThrow();
     });
 
     it("handles json option", async () => {
-      const { stderr } = await ctx.runEask("analyze --json");
-      await ctx.runEask("analyze Eask --json");
-
-      // try to parse output, errors if not valid
-      JSON.parse(stderr);
+      // alternate command order
+      await expect(ctx.runEask("analyze Eask --json")).rejects.toThrow();
+      try {
+        await ctx.runEask("analyze --json");
+      } catch (e) {
+        tryJSON(e.stderr);
+        const res = ctx.errorToCommandOutput(e);
+        const resClean = res.sanitized().raw();
+        expect(resClean).toMatchSnapshot();
+      }
     });
 
     it("matches snapshot", async () => {
-      const res = await ctx.runEask("analyze");
-      const resClean = res.sanitized().raw();
-      expect(resClean).toMatchSnapshot();
+      try {
+        await ctx.runEask("analyze");
+        expect.failing();
+      } catch (e) {
+        expect(e.code).toBe(1);
+        const res = ctx.errorToCommandOutput(e);
+        const resClean = res.sanitized().raw();
+        expect(resClean).toMatchSnapshot();
+      }
     });
 
     it("should report multiple definitions", async () => {
-      const { stderr } = await ctx.runEask("analyze");
-      // expect this substring
-      expect(stderr).toMatch("Multiple definition of `package'");
+      try {
+        await ctx.runEask("analyze");
+        expect.failing();
+      } catch ({ stderr }) {
+        // expect this substring
+        expect(stderr).toMatch("Multiple definition of `package'");
+      }
     });
   });
 
@@ -38,17 +84,94 @@ describe("analyze", () => {
     });
   });
 
-  describe("in ./metadata", () => {
-    const ctx = new TestContext("./test/jest/metadata");
+  describe("in ./analyze/metadata", () => {
+    const ctx = new TestContext("./test/jest/analyze/metadata");
 
     it("handles plain text", async () => {
-      await ctx.runEask("analyze");
-      await ctx.runEask("analyze Eask");
+      await expect(ctx.runEask("analyze")).rejects.toThrow();
+      await expect(ctx.runEask("analyze Eask")).rejects.toThrow(
+        expect.objectContaining({
+          code: 1,
+          stderr: expect.stringContaining("(Checked 1 file)"),
+        }),
+      );
     });
 
     it("handles json", async () => {
-      await ctx.runEask("analyze --json");
-      await ctx.runEask("analyze Eask --json");
+      await expect(ctx.runEask("analyze --json")).rejects.toThrow();
+      await expect(ctx.runEask("analyze Eask --json")).rejects.toMatchObject({
+        code: 1,
+        stderr: expect.stringContaining("(Checked 1 file)"),
+      });
+    });
+  });
+
+  describe("in ./analyze/errors", () => {
+    const ctx = new TestContext("./test/jest/analyze/errors");
+
+    // Eask-normal - no errors or warnings
+    // Eask-warn   - only warnings
+    // Eask-error  - errors and  warnings
+
+    it("should check Eask-normal", async () => {
+      await ctx.runEask("analyze Eask-normal");
+    });
+
+    it("should check Eask-warn", async () => {
+      await ctx.runEask("analyze Eask-warn");
+    });
+
+    it("should error on Eask-errors", async () => {
+      await expect(ctx.runEask("analyze Eask-error")).rejects.toThrow();
+    });
+
+    // JSON
+    it("handles json option when no errors", async () => {
+      const { stderr } = await ctx.runEask("analyze --json Eask-normal");
+      tryJSON(stderr);
+    });
+
+    it("handles json option when lexical binding warnings are present", async () => {
+      const { stderr } = await ctx.runEask("analyze --json Eask-lexical");
+      tryJSON(stderr);
+    });
+
+    // --strict and --allow-error
+    it("should error when using --strict on Eask-warn", async () => {
+      await expect(ctx.runEask("analyze --strict Eask-warn")).rejects.toThrow();
+    });
+
+    // sanity check: flag should not change behavior in this case
+    // this is because the menaing of --allow-error is "continue to the end"
+    it("should error when --allow-error is set", async () => {
+      await expect(
+        ctx.runEask("analyze Eask-error --allow-error"),
+      ).rejects.toThrow();
+    });
+
+    it("should print Checked when there are errors", async () => {
+      await expect(ctx.runEask("analyze Eask-error")).rejects.toMatchObject({
+        stderr: expect.stringContaining("(Checked 1 file)"),
+      });
+    });
+
+    it("should check all files when --allow-error is set", async () => {
+      // this is not a great test because when there is any error it doesn't print this note
+      await expect(
+        ctx.runEask("analyze --allow-error Eask-normal Eask-error"),
+      ).rejects.toMatchObject({
+        stderr: expect.stringContaining("(Checked 2 files)"),
+      });
+    });
+
+    // although the output does match, it still doesn't exit with an error
+    it("should have later warnings when --allow-error is set", async () => {
+      await expect(
+        ctx.runEask("analyze --allow-error Eask-error Eask-warn"),
+        // this warning is specific to Eask-warn
+      ).rejects.toMatchObject({
+        stderr: expect.stringContaining("missing `none.el'"),
+      });
     });
   });
 });
diff --git a/test/jest/dsl/.gitignore b/test/jest/analyze/dsl/.gitignore
similarity index 100%
rename from test/jest/dsl/.gitignore
rename to test/jest/analyze/dsl/.gitignore
diff --git a/test/jest/dsl/Eask b/test/jest/analyze/dsl/Eask
similarity index 100%
rename from test/jest/dsl/Eask
rename to test/jest/analyze/dsl/Eask
diff --git a/test/jest/dsl/check-dsl.el b/test/jest/analyze/dsl/check-dsl.el
similarity index 100%
rename from test/jest/dsl/check-dsl.el
rename to test/jest/analyze/dsl/check-dsl.el
diff --git a/test/jest/analyze/errors/Eask-error b/test/jest/analyze/errors/Eask-error
new file mode 100644
index 00000000..3d2cb3f3
--- /dev/null
+++ b/test/jest/analyze/errors/Eask-error
@@ -0,0 +1,2 @@
+(package "" "" "")
+(scrog "unrecognized")
diff --git a/test/jest/analyze/errors/Eask-lexical b/test/jest/analyze/errors/Eask-lexical
new file mode 100644
index 00000000..1b2c2e27
--- /dev/null
+++ b/test/jest/analyze/errors/Eask-lexical
@@ -0,0 +1,2 @@
+(package "check-dsl" "0.0.1" "Test for DSL")
+(keywords "dsl")
diff --git a/test/jest/analyze/errors/Eask-normal b/test/jest/analyze/errors/Eask-normal
new file mode 100644
index 00000000..00169f2d
--- /dev/null
+++ b/test/jest/analyze/errors/Eask-normal
@@ -0,0 +1,3 @@
+;; -*- mode: eask; lexical-binding: t -*-
+(package "check-dsl" "0.0.1" "Test for DSL")
+(keywords "dsl")
diff --git a/test/jest/analyze/errors/Eask-warn b/test/jest/analyze/errors/Eask-warn
new file mode 100644
index 00000000..9fdb2623
--- /dev/null
+++ b/test/jest/analyze/errors/Eask-warn
@@ -0,0 +1,5 @@
+(package "check-dsl" "0.0.1" "Test for DSL")
+
+(keywords "dsl")
+
+(package-file "none.el")
diff --git a/test/jest/metadata/.gitignore b/test/jest/analyze/metadata/.gitignore
similarity index 100%
rename from test/jest/metadata/.gitignore
rename to test/jest/analyze/metadata/.gitignore
diff --git a/test/jest/metadata/Eask b/test/jest/analyze/metadata/Eask
similarity index 100%
rename from test/jest/metadata/Eask
rename to test/jest/analyze/metadata/Eask
diff --git a/test/jest/metadata/check-metadata.el b/test/jest/analyze/metadata/check-metadata.el
similarity index 100%
rename from test/jest/metadata/check-metadata.el
rename to test/jest/analyze/metadata/check-metadata.el
diff --git a/test/jest/helpers.js b/test/jest/helpers.js
index 31980c8f..d23488af 100644
--- a/test/jest/helpers.js
+++ b/test/jest/helpers.js
@@ -241,6 +241,10 @@ class TestContext {
         .catch(() => {}); // ignore "does not exist errors"
     }
   }
+
+  errorToCommandOutput(e) {
+    return new CommandOutput(e, this.cwd);
+  }
 }
 
 module.exports = {