Merge branch 'master' into sharding_fix

Author: ceallen

CHANGELOG.md

@@ -1,3 +1,9 @@
+0.5.2 (2023-??)
+------------------
+* Feature: Allow configuring error email addresses and email subject via UI. 
+* Bugfix: . and .. should now be allowed to be used when specifying the templates directory.
+* Bugfix: corrected cron schedule incorrectly shifting back one day upon save.
+
 0.5.1 (2023-02-22)
 ------------------
 

notebooker/constants.py

@@ -80,11 +80,13 @@ class NotebookResultBase(object):
     status = attr.ib(default=JobStatus.ERROR)
     overrides = attr.ib(default=attr.Factory(dict))
     mailto = attr.ib(default="")
+    error_mailto = attr.ib(default="")
     generate_pdf_output = attr.ib(default=True)
     hide_code = attr.ib(default=False)
     stdout = attr.ib(default=attr.Factory(list))
     scheduler_job_id = attr.ib(default=None)
     mailfrom = attr.ib(default=None)
+    email_subject = attr.ib(default=None)
     is_slideshow = attr.ib(default=False)
 
     def saveable_output(self):
@@ -100,6 +102,7 @@ class NotebookResultPending(NotebookResultBase):
     report_title = attr.ib(default="")
     overrides = attr.ib(default=attr.Factory(dict))
     mailto = attr.ib(default="")
+    error_mailto = attr.ib(default="")
     generate_pdf_output = attr.ib(default=True)
     hide_code = attr.ib(default=False)
     scheduler_job_id = attr.ib(default=None)
@@ -115,15 +118,13 @@ class NotebookResultError(NotebookResultBase):
     report_title = attr.ib(default="")
     overrides = attr.ib(default=attr.Factory(dict))
     mailto = attr.ib(default="")
+    error_mailto = attr.ib(default="")
     generate_pdf_output = attr.ib(default=True)
     hide_code = attr.ib(default=False)
     scheduler_job_id = attr.ib(default=None)
     mailfrom = attr.ib(default=None)
     is_slideshow = attr.ib(default=False)
-
-    @property
-    def email_subject(self):
-        return ""
+    email_subject = attr.ib(default=None)
 
     @property
     def raw_html(self):
@@ -155,6 +156,7 @@ class NotebookResultComplete(NotebookResultBase):
     report_title = attr.ib(default="")
     overrides = attr.ib(default=attr.Factory(dict))
     mailto = attr.ib(default="")
+    error_mailto = attr.ib(default="")
     email_subject = attr.ib(default="")
     generate_pdf_output = attr.ib(default=True)
     hide_code = attr.ib(default=False)
@@ -185,6 +187,7 @@ def saveable_output(self):
             "job_start_time": self.job_start_time,
             "job_finish_time": self.job_finish_time,
             "mailto": self.mailto,
+            "error_mailto": self.error_mailto,
             "email_subject": self.email_subject,
             "overrides": self.overrides,
             "generate_pdf_output": self.generate_pdf_output,
@@ -200,9 +203,9 @@ def __repr__(self):
         return (
             "NotebookResultComplete(job_id={job_id}, status={status}, report_name={report_name}, "
             "job_start_time={job_start_time}, job_finish_time={job_finish_time}, update_time={update_time}, "
-            "report_title={report_title}, overrides={overrides}, mailto={mailto}, mailfrom={mailfrom}"
-            "email_subject={email_subject}, generate_pdf_output={generate_pdf_output}, hide_code={hide_code}, "
-            "scheduler_job_id={scheduler_job_id}, is_slideshow={is_slideshow})".format(
+            "report_title={report_title}, overrides={overrides}, mailto={mailto}, error_mailto={error_mailto}, "
+            "mailfrom={mailfrom}, email_subject={email_subject}, generate_pdf_output={generate_pdf_output}, "
+            "hide_code={hide_code}, scheduler_job_id={scheduler_job_id}, is_slideshow={is_slideshow})".format(
                 job_id=self.job_id,
                 status=self.status,
                 report_name=self.report_name,
@@ -212,6 +215,7 @@ def __repr__(self):
                 report_title=self.report_title,
                 overrides=self.overrides,
                 mailto=self.mailto,
+                error_mailto=self.error_mailto,
                 mailfrom=self.mailfrom,
                 email_subject=self.email_subject,
                 generate_pdf_output=self.generate_pdf_output,

notebooker/execute_notebook.py

@@ -44,6 +44,7 @@ def _run_checks(
     generate_pdf_output: Optional[bool] = True,
     hide_code: Optional[bool] = False,
     mailto: Optional[str] = "",
+    error_mailto: Optional[str] = "",
     email_subject: Optional[str] = "",
     prepare_only: Optional[bool] = False,
     notebooker_disable_git: bool = False,
@@ -76,7 +77,9 @@ def _run_checks(
     generate_pdf_output : `Optional[bool]`
         Whether to generate PDF output or not. NB this requires xelatex to be installed on the executor.
     mailto : `Optional[str]`
-        Comma-separated email addresses to send on completion (or error).
+        Comma-separated email addresses to send on completion.
+    error_mailto : `Optional[str]`
+        Comma-separated email addresses to send on error.
     prepare_only : `Optional[bool]`
         Internal usage. Whether we want to do everything apart from executing the notebook.
     scheduler_job_id : `Optional[str]`
@@ -128,6 +131,7 @@ def _run_checks(
         raw_html=html,
         email_html=email_html,
         mailto=mailto,
+        error_mailto=error_mailto,
         email_subject=email_subject,
         pdf=pdf,
         generate_pdf_output=generate_pdf_output,
@@ -191,6 +195,7 @@ def run_report(
             template_base_dir,
             overrides,
             mailto=mailto,
+            error_mailto=error_mailto,
             email_subject=email_subject,
             generate_pdf_output=generate_pdf_output,
             hide_code=hide_code,
@@ -215,12 +220,14 @@ def run_report(
             report_title=report_title,
             error_info=error_info,
             overrides=overrides,
-            mailto=error_mailto or mailto,
+            mailto=mailto,
+            error_mailto=error_mailto,
             generate_pdf_output=generate_pdf_output,
             scheduler_job_id=scheduler_job_id,
             mailfrom=mailfrom,
             hide_code=hide_code,
             is_slideshow=is_slideshow,
+            email_subject=email_subject,
         )
         logger.error(
             "Report run failed. Saving error result to mongo library %s@%s...",
@@ -402,8 +409,7 @@ def execute_notebook_entrypoint(
             mailfrom=mailfrom,
             is_slideshow=is_slideshow,
         )
-        if result.mailto:
-            send_result_email(result, config.DEFAULT_MAILFROM)
+        send_result_email(result, config.DEFAULT_MAILFROM)
         logger.info(f"Here is the result!{result}")
         if isinstance(result, NotebookResultError):
             logger.warning("Notebook execution failed! Output was:")
@@ -458,6 +464,7 @@ def run_report_in_subprocess(
     report_name,
     report_title,
     mailto,
+    error_mailto,
     overrides,
     *,
     hide_code=False,
@@ -466,6 +473,7 @@ def run_report_in_subprocess(
     scheduler_job_id=None,
     run_synchronously=False,
     mailfrom=None,
+    email_subject=None,
     n_retries=3,
     is_slideshow=False,
 ) -> str:
@@ -476,16 +484,20 @@ def run_report_in_subprocess(
     :param report_name: `str` The report which we are executing
     :param report_title: `str` The user-specified title of the report
     :param mailto: `Optional[str]` Who the results will be emailed to
+    :param error_mailto: `Optional[str]` Who the errors will be emailed to
     :param overrides: `Optional[Dict[str, Any]]` The parameters to be passed into the report
     :param generate_pdf_output: `bool` Whether we're generating a PDF. Defaults to False.
     :param prepare_only: `bool` Whether to do everything except execute the notebook. Useful for testing.
     :param scheduler_job_id: `Optional[str]` if the job was triggered from the scheduler, this is the scheduler's job id
     :param run_synchronously: `bool` If True, then we will join the stderr monitoring thread until the job has completed
     :param mailfrom: `str` if passed, then this string will be used in the from field
+    :param email_subject: `str` if passed, then this string will be used in the email subject
     :param n_retries: The number of retries to attempt.
     :param is_slideshow: Whether the notebook is a reveal.js slideshow or not.
     :return: The unique job_id.
     """
+    if error_mailto is None:
+        error_mailto = ""
     job_id = str(uuid.uuid4())
     job_start_time = datetime.datetime.now()
     result_serializer = initialize_serializer_from_config(base_config)
@@ -497,10 +509,12 @@ def run_report_in_subprocess(
         status=JobStatus.SUBMITTED,
         overrides=overrides,
         mailto=mailto,
+        error_mailto=error_mailto,
         generate_pdf_output=generate_pdf_output,
         hide_code=hide_code,
         scheduler_job_id=scheduler_job_id,
         is_slideshow=is_slideshow,
+        email_subject=email_subject,
     )
 
     command = (
@@ -530,6 +544,8 @@ def run_report_in_subprocess(
             report_title,
             "--mailto",
             mailto,
+            "--error-mailto",
+            error_mailto,
             "--overrides-as-json",
             json.dumps(overrides),
             "--pdf-output" if generate_pdf_output else "--no-pdf-output",
@@ -541,6 +557,7 @@ def run_report_in_subprocess(
         + (["--is-slideshow"] if is_slideshow else [])
         + ([f"--scheduler-job-id={scheduler_job_id}"] if scheduler_job_id is not None else [])
         + ([f"--mailfrom={mailfrom}"] if mailfrom is not None else [])
+        + ([f"--email-subject={email_subject}"] if email_subject else [])
     )
     p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 

notebooker/serialization/mongo.py

@@ -187,11 +187,7 @@ def update_check_status(self, job_id: str, status: JobStatus, **extra):
             existing["status"] = status.value
             for k, v in extra.items():
                 if k == "error_info" and v:
-                    self.result_data_store.put(
-                        v,
-                        filename=_error_info_filename(job_id),
-                        encoding="utf-8",
-                    )
+                    self.result_data_store.put(v, filename=_error_info_filename(job_id), encoding="utf-8")
                 else:
                     existing[k] = v
             self._save_raw_to_db(existing)
@@ -205,10 +201,12 @@ def save_check_stub(
         status: JobStatus = JobStatus.PENDING,
         overrides: Optional[Dict] = None,
         mailto: str = "",
+        error_mailto: str = "",
         generate_pdf_output: bool = True,
         hide_code: bool = False,
         scheduler_job_id: Optional[str] = None,
         is_slideshow: bool = False,
+        email_subject: Optional[str] = None,
     ) -> None:
         """Call this when we are just starting a check. Saves a "pending" job into storage."""
         job_start_time = job_start_time or datetime.datetime.now()
@@ -220,6 +218,8 @@ def save_check_stub(
             job_start_time=job_start_time,
             report_name=report_name,
             mailto=mailto,
+            error_mailto=error_mailto,
+            email_subject=email_subject,
             generate_pdf_output=generate_pdf_output,
             overrides=overrides or {},
             hide_code=hide_code,
@@ -242,9 +242,7 @@ def save_check_result(self, notebook_result: Union[NotebookResultComplete, Noteb
                     filename=filename_func(notebook_result.job_id),
                     encoding="utf-8",
                 )
-        for json_attribute, filename_func in [
-            ("raw_ipynb_json", _raw_json_filename),
-        ]:
+        for json_attribute, filename_func in [("raw_ipynb_json", _raw_json_filename)]:
             if getattr(notebook_result, json_attribute, None):
                 self.result_data_store.put(
                     json.dumps(getattr(notebook_result, json_attribute)),
@@ -317,10 +315,12 @@ def _convert_result(
                 generate_pdf_output=result.get("generate_pdf_output", True),
                 report_title=result.get("report_title", result["report_name"]),
                 mailto=result.get("mailto", ""),
+                error_mailto=result.get("error_mailto", ""),
                 hide_code=result.get("hide_code", False),
                 stdout=result.get("stdout", []),
                 scheduler_job_id=result.get("scheduler_job_id", None),
                 is_slideshow=result.get("is_slideshow", False),
+                email_subject=result.get("email_subject", None),
             )
         elif cls == NotebookResultPending:
             return NotebookResultPending(
@@ -333,6 +333,8 @@ def _convert_result(
                 generate_pdf_output=result.get("generate_pdf_output", True),
                 report_title=result.get("report_title", result["report_name"]),
                 mailto=result.get("mailto", ""),
+                error_mailto=result.get("error_mailto", ""),
+                email_subject=result.get("email_subject", ""),
                 hide_code=result.get("hide_code", False),
                 stdout=result.get("stdout", []),
                 scheduler_job_id=result.get("scheduler_job_id", None),
@@ -356,6 +358,8 @@ def _convert_result(
                 generate_pdf_output=result.get("generate_pdf_output", True),
                 report_title=result.get("report_title", result["report_name"]),
                 mailto=result.get("mailto", ""),
+                error_mailto=result.get("error_mailto", ""),
+                email_subject=result.get("email_subject", ""),
                 hide_code=result.get("hide_code", False),
                 stdout=result.get("stdout", []),
                 scheduler_job_id=result.get("scheduler_job_id", False),

notebooker/utils/notebook_execution.py

@@ -15,16 +15,13 @@ def _output_dir(output_base_dir, report_name, job_id):
     return os.path.join(output_base_dir, report_name, job_id)
 
 
-def send_result_email(result: Union[NotebookResultComplete, NotebookResultError], default_mailfrom: str) -> None:
-    if result.mailfrom:
-        mailfrom = result.mailfrom
-    else:
-        mailfrom = default_mailfrom
-    to_email = result.mailto
+def _send_email(from_email: str, to_email: str, result: Union[NotebookResultComplete, NotebookResultError]) -> None:
     report_title = (
         result.report_title.decode("utf-8") if isinstance(result.report_title, bytes) else result.report_title
     )
-    subject = result.email_subject or f"Notebooker: {report_title} report completed with status: {result.status.value}"
+    subject = f"Notebooker: {report_title} report completed with status: {result.status.value}"
+    if isinstance(result, NotebookResultComplete):
+        subject = result.email_subject or subject
     body = result.email_html or result.raw_html
     attachments = []
     tmp_dir = None
@@ -57,8 +54,23 @@ def send_result_email(result: Union[NotebookResultComplete, NotebookResultError]
         msg = ["Please either activate HTML emails, or see the PDF attachment.", body]
 
         logger.info("Sending email to %s with %d attachments", to_email, len(attachments))
-        mail(mailfrom, to_email, subject, msg, attachments=attachments)
+        mail(from_email, to_email, subject, msg, attachments=attachments)
     finally:
         if tmp_dir:
             logger.info("Cleaning up temporary email attachment directory %s", tmp_dir)
             shutil.rmtree(tmp_dir)
+
+
+def send_result_email(result: Union[NotebookResultComplete, NotebookResultError], default_mailfrom: str) -> None:
+    if result.mailfrom:
+        mailfrom = result.mailfrom
+    else:
+        mailfrom = default_mailfrom
+    if isinstance(result, NotebookResultComplete):
+        to_email = result.mailto
+    else:
+        to_email = result.error_mailto or result.mailto
+    if not to_email:
+        logger.info("Not sending email as no recipients specified")
+        return
+    _send_email(from_email=mailfrom, to_email=to_email, result=result)

notebooker/utils/templates.py

@@ -14,7 +14,7 @@
 
 
 def _valid_dirname(d):
-    hidden_dirs = [dir for dir in d.split("/") if dir.startswith(".") and (dir != "." or dir != "..")]
+    hidden_dirs = [dir for dir in d.split("/") if dir.startswith(".") and dir not in {".", ".."}]
     return "__init__" not in d and "__pycache__" not in d and not hidden_dirs