Rename field to useRobots

Author: tw4l

backend/btrixcloud/models.py

@@ -362,6 +362,8 @@ class RawCrawlConfig(BaseModel):
     combineWARC: Optional[bool] = None
 
     useSitemap: Optional[bool] = False
+    useRobots: Optional[bool] = False
+
     failOnFailedSeed: Optional[bool] = False
     failOnContentCheck: Optional[bool] = False
 
@@ -376,8 +378,6 @@ class RawCrawlConfig(BaseModel):
 
     saveStorage: Optional[bool] = False
 
-    robots: Optional[bool] = False
-
 
 # ============================================================================
 class CrawlConfigIn(BaseModel):

frontend/src/components/ui/config-details.ts

@@ -477,7 +477,7 @@ export class ConfigDetails extends BtrixElement {
       )}
       ${this.renderSetting(
         msg("Skip Pages Disallowed by robots.txt"),
-        Boolean(config.robots),
+        Boolean(config.useRobots),
       )}
       ${this.renderSetting(
         msg("Fail Crawl If Not Logged In"),
@@ -555,7 +555,7 @@ export class ConfigDetails extends BtrixElement {
       )}
       ${this.renderSetting(
         msg("Skip Pages Disallowed by robots.txt"),
-        Boolean(config.robots),
+        Boolean(config.useRobots),
       )}
       ${this.renderSetting(
         msg("Check For Sitemap"),

frontend/src/features/crawl-workflows/workflow-editor.ts

@@ -1064,11 +1064,11 @@ export class WorkflowEditor extends BtrixElement {
       `)}
       ${this.renderHelpTextCol(infoTextFor["includeLinkedPages"], false)}
       ${inputCol(html`
-        <sl-checkbox name="robots" ?checked=${this.formState.robots}>
+        <sl-checkbox name="useRobots" ?checked=${this.formState.useRobots}>
           ${msg("Skip pages disallowed by robots.txt")}
         </sl-checkbox>
       `)}
-      ${this.renderHelpTextCol(infoTextFor["robots"], false)}
+      ${this.renderHelpTextCol(infoTextFor["useRobots"], false)}
       ${inputCol(html`
         <sl-checkbox
           name="failOnContentCheck"
@@ -1587,11 +1587,11 @@ https://example.net`}
       `)}
       ${this.renderHelpTextCol(infoTextFor["includeLinkedPages"], false)}
       ${inputCol(html`
-        <sl-checkbox name="robots" ?checked=${this.formState.robots}>
+        <sl-checkbox name="useRobots" ?checked=${this.formState.useRobots}>
           ${msg("Skip pages disallowed by robots.txt")}
         </sl-checkbox>
       `)}
-      ${this.renderHelpTextCol(infoTextFor["robots"], false)}
+      ${this.renderHelpTextCol(infoTextFor["useRobots"], false)}
       ${inputCol(html`
         <sl-checkbox name="useSitemap" ?checked=${this.formState.useSitemap}>
           ${msg("Check for sitemap")}
@@ -3355,7 +3355,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
     | "failOnFailedSeed"
     | "failOnContentCheck"
     | "saveStorage"
-    | "robots"
+    | "useRobots"
   > {
     const jsonSeeds = this.formState.seedListFormat === SeedListFormat.JSON;
 
@@ -3375,7 +3375,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
       failOnFailedSeed: this.formState.failOnFailedSeed,
       failOnContentCheck: this.formState.failOnContentCheck,
       saveStorage: this.formState.saveStorage,
-      robots: this.formState.robots,
+      useRobots: this.formState.useRobots,
     };
 
     return config;
@@ -3389,7 +3389,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
     | "failOnFailedSeed"
     | "failOnContentCheck"
     | "saveStorage"
-    | "robots"
+    | "useRobots"
   > {
     const primarySeedUrl = this.formState.primarySeedUrl;
     const includeUrlList = this.formState.customIncludeUrlList
@@ -3422,7 +3422,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
       failOnFailedSeed: false,
       failOnContentCheck: this.formState.failOnContentCheck,
       saveStorage: this.formState.saveStorage,
-      robots: this.formState.robots,
+      useRobots: this.formState.useRobots,
     };
     return config;
   }

frontend/src/strings/crawl-workflows/infoText.ts

@@ -85,7 +85,7 @@ export const infoTextFor = {
   saveStorage: msg(
     `Include data from the browser's local and session storage in the web archive.`,
   ),
-  robots: msg(
+  useRobots: msg(
     `Check for a robots.txt file for each host and skip any disallowed pages.`,
   ),
 } as const satisfies Partial<Record<Field, string | TemplateResult>>;

frontend/src/types/crawler.ts

@@ -55,7 +55,7 @@ export type SeedConfig = Expand<
     customBehaviors: string[];
     clickSelector: string;
     saveStorage?: boolean;
-    robots?: boolean;
+    useRobots?: boolean;
   }
 >;
 

frontend/src/utils/workflow.ts

@@ -184,7 +184,7 @@ export type FormState = {
   selectLinks: string[];
   clickSelector: string;
   saveStorage: WorkflowParams["config"]["saveStorage"];
-  robots: WorkflowParams["config"]["robots"];
+  useRobots: WorkflowParams["config"]["useRobots"];
 };
 
 export type FormStateField = keyof FormState;
@@ -247,7 +247,7 @@ export const getDefaultFormState = (): FormState => ({
   clickSelector: DEFAULT_AUTOCLICK_SELECTOR,
   customBehavior: false,
   saveStorage: false,
-  robots: false,
+  useRobots: false,
 });
 
 export const mapSeedToUrl = (arr: Seed[]) =>
@@ -421,7 +421,7 @@ export function getInitialFormState(params: {
       params.initialWorkflow.crawlerChannel || defaultFormState.crawlerChannel,
     proxyId: params.initialWorkflow.proxyId || defaultFormState.proxyId,
     saveStorage: params.initialWorkflow.config.saveStorage,
-    robots: params.initialWorkflow.config.robots,
+    useRobots: params.initialWorkflow.config.useRobots,
     ...formState,
   };
 }