Update the lists (#487)

* update the lists
* update the script
* fix tests

Author: atuchin-m

data/brave/brave-main-list.txt

@@ -7,7 +7,8 @@ const args = process.argv.slice(2);
 
 if (args.length < 2) {
   console.error(
-    "Usage: node update-lists.js <Brave Services Key> <target version for brave list (i.e. 1.0.10268)>"
+    "Usage: node update-lists.js <Brave Services Key> <target version for brave list (i.e. 1.0.10268)>\n" +
+      "The component name is 'Brave Ad Block Updater'"
   );
   process.exit(1);
 }
@@ -28,20 +29,32 @@ execSync(
   "curl -o data/easylist.to/easylistgermany/easylistgermany.txt https://easylist.to/easylistgermany/easylistgermany.txt"
 );
 
-execSync(
-  `curl -o extension.zip -H "BraveServiceKey: ${apiKey}" ` +
-    `https://brave-core-ext.s3.brave.com/release/${extensionId}/extension_${versionNumber}.crx`
+const rootDir = path.join(__dirname, "..");
+const tempDir = path.resolve(
+  fs.mkdtempSync("temp-brave-list", {
+    dir: rootDir,
+  })
 );
 
-const tempDir = fs.mkdtempSync("temp-brave-list");
-const listPath = path.join(tempDir, "list.txt");
 try {
-  execSync("unzip extension.zip -d " + tempDir);
-} catch (e) {
-  if (!fs.existsSync(listPath)) {
-    console.error("Failed to find list.txt in extension.zip");
-    process.exit(1);
+  process.chdir(tempDir);
+
+  execSync(
+    `curl -o extension.zip -H "BraveServiceKey: ${apiKey}" ` +
+      `https://brave-core-ext.s3.brave.com/release/${extensionId}/extension_${versionNumber}.crx`
+  );
+
+  const listPath = path.join(tempDir, "list.txt");
+  try {
+    execSync("unzip extension.zip -d .");
+  } catch (e) {
+    // .crx is not a zip file, so we expect an error here.
+    if (!fs.existsSync(listPath)) {
+      throw new Error("Failed to find list.txt in extension.zip");
+    }
   }
-}
 
-execSync(`mv -f ${listPath} data/brave/brave-main-list.txt`);
+  fs.renameSync(listPath, path.join(rootDir, "data/brave/brave-main-list.txt"));
+} finally {
+  fs.rmdirSync(tempDir, { recursive: true });
+}

data/easylist.to/easylist/easylist.txt

@@ -202,5 +202,5 @@ fn check_rule_matching_browserlike() {
     let (blocked, passes) = bench_rule_matching_browserlike(&engine, &requests);
     let msg = "The number of blocked/passed requests has changed. ".to_string()
         + "If this is expected, update the expected values in the test.";
-    assert_eq!((blocked, passes), (103973, 138972), "{}", msg);
+    assert_eq!((blocked, passes), (101756, 141189), "{}", msg);
 }

data/easylist.to/easylist/easyprivacy.txt

@@ -1368,30 +1368,27 @@ mod legacy_rule_parsing_tests {
         }
     }
 
-    // number of expected EasyList cosmetic rules from old engine is 31144, but is incorrect as it skips a few particularly long rules that are nevertheless valid
-    // easyList = { 24478, 31144, 0, 5589 };
-    // not handling (and not including) filters with the following options:
-    // - $popup
-    // - $elemhide
-    // difference from original counts caused by not handling document/subdocument options and possibly miscounting on the blocker side.
-    // Printing all non-cosmetic, non-html, non-comment/-empty rules and ones with no unsupported options yields 29142 items
-    // This engine also handles 3 rules that old one does not
+    // The number of loaded rules differs from the text files due to:
+    // * not handling (and not including) filters with the following options:
+    //   - $popup
+    //   - $elemhide
+    // * not handling document/subdocument options;
+    // * the optimizer that merges multiple rules into one;
     const EASY_LIST: ListCounts = ListCounts {
-        filters: 35597, // 36259 - 662 exceptions
+        filters: 35043,
         cosmetic_filters: if cfg!(feature = "css-validation") {
-            23072
+            23668
         } else {
-            23080
+            23682
         },
-        exceptions: 662,
+        exceptions: 706,
         duplicates: 0,
     };
-    // easyPrivacy = { 11817, 0, 0, 1020 };
     // differences in counts explained by hashset size underreporting as detailed in the next two cases
     const EASY_PRIVACY: ListCounts = ListCounts {
-        filters: 52278, // 52998 - 720 exceptions
-        cosmetic_filters: 21,
-        exceptions: 720,
+        filters: 53029, // total - exceptions
+        cosmetic_filters: 28,
+        exceptions: 737,
         duplicates: 2,
     };
     // ublockUnbreak = { 4, 8, 0, 94 };

data/easylist.to/easylistgermany/easylistgermany.txt

@@ -219,9 +219,9 @@ mod tests {
         let data = engine.serialize().unwrap();
 
         let expected_hash = if cfg!(feature = "css-validation") {
-            6718506180720782170
+            7254547691107602751
         } else {
-            6839468684492187294
+            4130628479730907288
         };
 
         assert_eq!(hash(&data), expected_hash, "{}", HASH_MISMATCH_MSG);