feat: make preview possible in development mode

Author: alvis

README.md

@@ -29,6 +29,7 @@ Tired of uploading a markdown file to your GitHub for every new blog post? Havin
 - ⌨️ Title and their properties in plain text accessible via the front matter
 - 🔮 All raw properties accessible via GraphQL
 - 🍻 Support for `remark` and `mdx`
+- 👀 Near real-time preview in development mode
 
 # Quick Start
 
@@ -213,9 +214,38 @@ interface PluginConfig {
   databases?: string[];
   /** UUID of pages to be sourced, default to be `[]` i.e. none */
   pages?: string[];
+  /** the number of api calls per seconds allowed for preview, 0 to disable preview default to be 2.5 */
+  previewCallRate?: number;
+  /** TTL settings for each API call types, default to cache database metadata and blocks */
+  previewTTL?: {
+    /** the number of seconds in which a database metadata will be cached, default to be 0 i.e. permanent */
+    databaseMeta?: number;
+    /** the number of seconds in which a metadata of a database's entries will be cached, default to be 0.5 */
+    databaseEntries?: number;
+    /** the number of seconds in which a page metadata will be cached, default to be 0.5 */
+    pageMeta?: number;
+    /** the number of seconds in which a page content will be cached, default to be 0 i.e. permanent */
+    pageContent?: number;
+  };
 }
 ```
 
+# Preview Mode
+
+This plugin ships with a preview mode by default and it is enabled.
+Start your development server and type on your Notion page to see the content get updated on the Gatsby website.
+
+Under the hood, this plugin automatically pulls the page metadata from Notion regularly and checks for any updates using the `last_edited_time` property.
+When a change is detected, this plugin will reload the content automatically.
+
+**NOTE** To adjust the frequency of update, you can specify the maximum allowed number of API calls.
+The higher the more frequently it checks for updates.
+The actual frequency will be computed automatically according to your needs but be mindful of current limits for Notion API which is 3 requests per second at time of publishing.
+
+**NOTE** Unlike other integrations with preview, such as `gatsby-source-sanity`, this plugin can't sync any content from your Notion document that wasn't saved.
+Notion has autosaving, but it is delayed so you might not see an immediate change in preview.
+Don't worry though, because it’s only a matter of time before you see the change.
+
 # Known Limitations
 
 As this plugin relies on the the official Notion API which is still in beta, we share the same limitations as the API.
@@ -260,6 +290,7 @@ You just need to embed them using the normal markdown syntax as part of your par
 
 3. What can I do if I don't want to permanently delete a post but just hide it for awhile?
    You can create a page property (for example, a publish double checkbox) and use this information in your page creation process.
+   If you're in the development mode with preview enabled, you should be able to see the removal in near real-time.
 
 # About
 

source/gatsby-node.ts

@@ -16,7 +16,7 @@
 import { version as gatsbyVersion } from 'gatsby/package.json';
 
 import { name } from '#.';
-import { normaliseConfig, sync } from '#plugin';
+import { computePreviewUpdateInterval, normaliseConfig, sync } from '#plugin';
 
 import type { GatsbyNode } from 'gatsby';
 
@@ -31,6 +31,14 @@ export const pluginOptionsSchema: NonNullable<
     version: joi.string().optional(),
     databases: joi.array().items(joi.string()).optional(),
     pages: joi.array().items(joi.string()).optional(),
+    ttl: joi
+      .object({
+        databaseMeta: joi.number().optional(),
+        databaseEntries: joi.number().optional(),
+        pageMeta: joi.number().optional(),
+        pageContent: joi.number().optional(),
+      })
+      .optional(),
   });
 };
 
@@ -47,6 +55,26 @@ export const onPreBootstrap: NonNullable<GatsbyNode['onPreBootstrap']> = async (
   }
 };
 
+export const onCreateDevServer: NonNullable<GatsbyNode['onCreateDevServer']> =
+  async (args, partialConfig) => {
+    const pluginConfig = normaliseConfig(partialConfig);
+    const { previewCallRate } = pluginConfig;
+
+    const previewUpdateInterval = computePreviewUpdateInterval(pluginConfig);
+    if (previewCallRate && previewUpdateInterval) {
+      const scheduleUpdate = (): NodeJS.Timeout =>
+        setTimeout(async () => {
+          // sync entities from notion
+          await sync(args, pluginConfig);
+
+          // schedule the next update
+          scheduleUpdate();
+        }, previewUpdateInterval);
+
+      scheduleUpdate();
+    }
+  };
+
 export const sourceNodes: NonNullable<GatsbyNode['sourceNodes']> = async (
   args,
   partialConfig,

source/node.ts

@@ -103,7 +103,10 @@ export class NodeManager {
       this.createNode(this.nodifyEntity(entity));
     }
 
-    this.reporter.info(`[${name}] added ${added.length} nodes`);
+    // don't be noisy if there's nothing new happen
+    if (added.length > 0) {
+      this.reporter.info(`[${name}] added ${added.length} nodes`);
+    }
   }
 
   /**
@@ -115,7 +118,10 @@ export class NodeManager {
       this.createNode(this.nodifyEntity(entity));
     }
 
-    this.reporter.info(`[${name}] updated ${updated.length} nodes`);
+    // don't be noisy if there's nothing new happen
+    if (updated.length > 0) {
+      this.reporter.info(`[${name}] updated ${updated.length} nodes`);
+    }
   }
 
   /**
@@ -127,7 +133,10 @@ export class NodeManager {
       this.deleteNode(this.nodifyEntity(entity));
     }
 
-    this.reporter.info(`[${name}] removed ${removed.length} nodes`);
+    // don't be noisy if there's nothing new happen
+    if (removed.length > 0) {
+      this.reporter.info(`[${name}] removed ${removed.length} nodes`);
+    }
   }
 
   /**

source/plugin.ts

@@ -13,12 +13,12 @@
  * -------------------------------------------------------------------------
  */
 
-import { Notion } from '#client';
+import { DEFAULT_TTL, Notion } from '#client';
 import { NodeManager } from '#node';
 
 import type { NodePluginArgs, PluginOptions } from 'gatsby';
 
-import type { NotionOptions } from '#client';
+import type { NotionOptions, NotionTTL } from '#client';
 import type { FullDatabase, FullPage } from '#types';
 
 /** options for the source plugin */
@@ -27,11 +27,47 @@ export interface PluginConfig extends PluginOptions, NotionOptions {
   databases?: string[];
   /** id of pages to be sourced, default to be all shared pages */
   pages?: string[];
+  /** the number of api calls per seconds allowed for preview, 0 to disable preview default 2.5 */
+  previewCallRate?: number;
+  /** TTL settings for each API call types, default to cache database metadata and blocks */
+  previewTTL?: Partial<NotionTTL>;
 }
 
 interface FullPluginConfig extends PluginConfig {
   databases: string[];
   pages: string[];
+  previewCallRate: number;
+  previewTTL: NotionTTL;
+}
+
+const ONE_SECOND = 1000;
+
+const DEFAULT_PREVIEW_API_RATE = 2.5;
+
+/**
+ * compute the update interval for the preview mode
+ * @param pluginConfig the normalised plugin config
+ * @returns the number of milliseconds needed between each sync
+ */
+export function computePreviewUpdateInterval(
+  pluginConfig: FullPluginConfig,
+): number | null {
+  const { previewCallRate, databases, pages, previewTTL } = pluginConfig;
+
+  // it's minimum because if a page get edited, it will consume more
+  const minAPICallsNeededPerSync =
+    databases.length *
+      // get page title and properties etc.
+      ((previewTTL.databaseEntries !== 0 ? 1 : 0) +
+        // it will take 1 more if we want to keep database title and properties etc. up-to-date as well
+        (previewTTL.databaseMeta !== 0 ? 1 : 0)) +
+    pages.length *
+      // get page title and properties etc.
+      (previewTTL.pageMeta !== 0 ? 1 : 0);
+
+  const interval = (ONE_SECOND * minAPICallsNeededPerSync) / previewCallRate;
+
+  return interval > 0 ? interval : null;
 }
 
 /**
@@ -42,6 +78,8 @@ interface FullPluginConfig extends PluginConfig {
 export function normaliseConfig(
   config: Partial<PluginConfig>,
 ): FullPluginConfig {
+  const { previewCallRate = DEFAULT_PREVIEW_API_RATE } = config;
+
   const databases = [
     ...(config.databases ?? []),
     ...(process.env['GATSBY_NOTION_DATABASES']?.split(/, +/) ?? []),
@@ -58,7 +96,16 @@ export function normaliseConfig(
     (id) => !!id,
   );
 
-  return { ...config, databases, pages, plugins: [] };
+  const previewTTL = { ...DEFAULT_TTL, ...config.previewTTL };
+
+  return {
+    ...config,
+    databases,
+    pages,
+    previewCallRate,
+    previewTTL,
+    plugins: [],
+  };
 }
 
 /**

spec/gatsby-node.spec.ts

@@ -16,7 +16,11 @@
 import gatsbyPackage from 'gatsby/package.json';
 import joi from 'joi';
 
-import { pluginOptionsSchema, sourceNodes } from '#gatsby-node';
+import {
+  onCreateDevServer,
+  pluginOptionsSchema,
+  sourceNodes,
+} from '#gatsby-node';
 import { sync } from '#plugin';
 
 jest.mock('gatsby/package.json', () => {
@@ -73,6 +77,57 @@ describe('fn:onPreBootstrap', () => {
   it('fail with future gatsby after v3', testVersion('4.0.0', false));
 });
 
+describe('fn:onCreateDevServer', () => {
+  beforeEach(() => jest.clearAllMocks());
+  beforeEach(() => jest.useFakeTimers());
+  afterAll(() => jest.useRealTimers());
+
+  it('disable preview mode if the API rate is 0', async () => {
+    await onCreateDevServer(
+      {} as any,
+      {
+        previewCallRate: 0,
+        databases: ['database'],
+        pages: ['page'],
+        plugins: [],
+      },
+      jest.fn(),
+    );
+
+    jest.advanceTimersToNextTimer();
+
+    // not calling because it's disabled
+    expect(sync).toBeCalledTimes(0);
+  });
+
+  it('disable preview mode if no database or page is given', async () => {
+    await onCreateDevServer({} as any, { plugins: [] }, jest.fn());
+
+    jest.advanceTimersToNextTimer();
+
+    // not calling because it's disabled
+    expect(sync).toBeCalledTimes(0);
+  });
+
+  it('continuously sync data with Notion', async () => {
+    jest.clearAllMocks();
+    await onCreateDevServer(
+      {} as any,
+      { databases: ['database_continuous'], pages: ['page'], plugins: [] },
+      jest.fn(),
+    );
+
+    jest.advanceTimersToNextTimer();
+    await Promise.resolve();
+    expect(sync).toBeCalledTimes(1);
+
+    // sync again after a certain time
+    jest.advanceTimersToNextTimer();
+    await Promise.resolve();
+    expect(sync).toBeCalledTimes(2);
+  });
+});
+
 describe('fn:sourceNodes', () => {
   beforeEach(() => jest.clearAllMocks());
   beforeEach(() => jest.useFakeTimers());

spec/plugin.spec.ts

@@ -14,7 +14,13 @@
  */
 
 import { Notion } from '#client';
-import { getDatabases, getPages, normaliseConfig, sync } from '#plugin';
+import {
+  computePreviewUpdateInterval,
+  getDatabases,
+  getPages,
+  normaliseConfig,
+  sync,
+} from '#plugin';
 import { mockDatabase, mockPage } from './mock';
 
 const mockUpdate = jest.fn();
@@ -27,6 +33,66 @@ jest.mock('#node', () => ({
 
 const client = new Notion({ token: 'token' });
 
+describe('fn:computeUpdateInterval', () => {
+  it('compute the interval based on the total number of databases and pages', () => {
+    expect(
+      computePreviewUpdateInterval({
+        databases: ['database1', 'database2'],
+        pages: ['page1', 'page2'],
+        // 2 for database page query, 2 for page query
+        previewCallRate: 4,
+        // cache database title, but not pages' properties
+        previewTTL: {
+          databaseMeta: 0,
+          databaseEntries: 1,
+          pageMeta: 1,
+          pageContent: 0,
+        },
+        plugins: [],
+      }),
+    ).toEqual(1000);
+  });
+
+  it('increase the call demand if database title and properties etc. needed to be synchronised as well', () => {
+    expect(
+      computePreviewUpdateInterval({
+        databases: ['database1', 'database2'],
+        pages: ['page1', 'page2'],
+        // 2 for database meta, 2 for database page query, 2 for page query
+        previewCallRate: 6,
+        // cache database title, but not pages' properties
+        previewTTL: {
+          databaseMeta: 1,
+          databaseEntries: 1,
+          pageMeta: 1,
+          // NOTE pageContent doesn't make a difference here because it will be reloaded if the `last_edited_time` has changed
+          pageContent: 1,
+        },
+        plugins: [],
+      }),
+    ).toEqual(1000);
+  });
+
+  it('return null if everything is cached', () => {
+    expect(
+      computePreviewUpdateInterval({
+        databases: ['database1', 'database2'],
+        pages: ['page1', 'page2'],
+        // 2 for database meta, 2 for database page query, 2 for page query
+        previewCallRate: 100,
+        // cache database title, but not pages' properties
+        previewTTL: {
+          databaseMeta: 0,
+          databaseEntries: 0,
+          pageMeta: 0,
+          pageContent: 0,
+        },
+        plugins: [],
+      }),
+    ).toEqual(null);
+  });
+});
+
 describe('fn:normaliseConfig', () => {
   const env = { ...process.env };
   afterEach(() => (process.env = { ...env }));