feat: add robots.txt file

Author: byron-okta

Gruntfile.js

@@ -48,6 +48,10 @@ module.exports = (grunt) => {
                         src: ["sitemap.xml"],
                         dest: "dist/website/",
                     },
+                    {
+                        src: ["robots.txt"],
+                        dest: "dist/website/",
+                    },
                 ],
             },
             extension: {
@@ -278,4 +282,4 @@ module.exports = (grunt) => {
     grunt.registerTask("test", ["unit-tests", "functional-tests"]);
 
     grunt.registerTask("default", ["build-dev", "connect:website", "watch"]);
-};
+};

robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /beta/
+Allow: /
+Sitemap: https://jwt.io/sitemap.xml