Merge pull request #1473 from freeCodeCamp/mdn/yari

simon04 · web-flow · commit 22e776714529 · 2021-04-15T21:51:14.000+02:00
mdn: update scraper to mdn/yari
diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee
@@ -203,7 +203,7 @@ credits = [
     'https://raw.githubusercontent.com/apache/cordova-docs/master/LICENSE'
   ], [
     'CSS<br>DOM<br>HTTP<br>HTML<br>JavaScript<br>SVG<br>XPath',
-    '2005-2020 Mozilla and individual contributors',
+    '2005-2021 MDN contributors',
     'CC BY-SA',
     'https://creativecommons.org/licenses/by-sa/2.5/'
   ], [
diff --git a/assets/stylesheets/pages/_mdn.scss b/assets/stylesheets/pages/_mdn.scss
@@ -27,6 +27,7 @@
   p > code, li > code { @extend %label; }
 
   > .note,
+  .notecard, // MDN 2021
   .notice,
   .warning,
   .overheadIndicator,
diff --git a/lib/docs/filters/mdn/clean_html.rb b/lib/docs/filters/mdn/clean_html.rb
@@ -41,6 +41,18 @@ def call
           node.parent['id'] = node['name']
           node.before(node.content).remove
         end
+        css('h2 > a, h3 > a').each do |node|
+          node.parent.content = node.content
+        end
+
+        css('.notecard > h4').each do |node|
+          node.name = 'strong'
+        end
+
+        css('svg.deprecated').each do |node|
+          node.name = 'span'
+          node.content = node.content
+        end
 
         css('dt > a[id]').each do |node|
           next if node['href']
@@ -64,6 +76,9 @@ def call
         end
 
         # New compatibility tables
+        # FIXME(2021):
+        # - fetched from external JSON: https://developer.mozilla.org/en-US/docs/Web/SVG/Attribute/alignment-baseline/bcd.json
+        # - https://github.com/mdn/yari/blob/master/build/bcd-urls.js
 
         css('.bc-data #Legend + dl', '.bc-data #Legend', '.bc-data #Legend_2 + dl', '.bc-data #Legend_2', '.bc-browser-name').remove
 
diff --git a/lib/docs/filters/mdn/contribute_link.rb b/lib/docs/filters/mdn/contribute_link.rb
diff --git a/lib/docs/filters/svg/clean_html.rb b/lib/docs/filters/svg/clean_html.rb
@@ -11,7 +11,7 @@ def root
       end
 
       def other
-        css('.prevnext').remove
+        css('.prev-next').remove
 
         if at_css('p').content.include?("\u{00AB}")
           at_css('p').remove
diff --git a/lib/docs/scrapers/http.rb b/lib/docs/scrapers/http.rb
@@ -7,8 +7,6 @@ class Http < Mdn
 
     html_filters.push 'http/clean_html', 'http/entries', 'title'
 
-    options[:mdn_tag] = 'HTTP'
-
     options[:root_title] = 'HTTP'
     options[:title] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title }
     options[:container] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? '.content' : nil }
diff --git a/lib/docs/scrapers/mdn/css.rb b/lib/docs/scrapers/mdn/css.rb
@@ -6,8 +6,6 @@ class Css < Mdn
 
     html_filters.push 'css/clean_html', 'css/entries', 'title'
 
-    options[:mdn_tag] = 'CSS'
-
     options[:root_title] = 'CSS'
 
     options[:skip] = %w(/CSS3 /Media/Visual /paged_media /Media/TV /Media/Tactile)
diff --git a/lib/docs/scrapers/mdn/dom.rb b/lib/docs/scrapers/mdn/dom.rb
@@ -1,92 +1,12 @@
 module Docs
   class Dom < Mdn
-    prepend FixInternalUrlsBehavior
-    prepend FixRedirectionsBehavior
 
     self.name = 'DOM'
     self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'
 
     html_filters.push 'dom/clean_html', 'dom/entries', 'title'
 
-    options[:mdn_tag] = 'XSLT_Reference'
-
     options[:root_title] = 'DOM'
 
-    options[:skip] = %w(
-      /Reference
-      /Index
-      /Document_Object_Model
-      /document/createProcessingInstruction
-      /document/documentURIObject
-      /document/loadOverlay
-      /document/tooltipNode
-      /Document/cookie/Simple_document.cookie_framework
-      /DOMErrorHandler
-      /DOMLocator
-      /DOMObject
-      /DOMStringList
-      /Event/Comparison_of_Event_Targets
-      /Format
-      /IDBDatabaseException
-      /IndexedDB_API/Using_JavaScript_Generators_in_Firefox
-      /Notation
-      /ProcessingInstruction
-      /TypeInfo
-      /window/getAttention
-      /window/messageManager
-      /window/updateCommands
-      /window/pkcs11
-      /OES_texture_float)
-
-    options[:skip_patterns] = [
-      /NS/,
-      /XPC/,
-      /moz/i,
-      /gecko/i,
-      /webkit/i,
-      /gamepad/i,
-      /UserData/,
-      /Bluetooth/,
-      /FMRadio/i,
-      /XDomainRequest/i,
-      /\A\/Camera/,
-      /\A\/Data_Store_API/,
-      /\A\/DataStore/,
-      /\A\/DeviceStorage/,
-      /\A\/DocumentTouch/,
-      /\A\/document\/xml/,
-      /\A\/XMLDocument/,
-      /\A\/DOMCursor/,
-      /\A\/DOMRequest/,
-      /\A\/InstallTrigger/,
-      /\A\/Entity/,
-      /\A\/Settings/,
-      /telephony/i,
-      /\A\/NFC_API/,
-      /\A\/Window\/\w+bar/i,
-      /\A\/Apps/,
-      /\A\/Contact/,
-      /\A\/L10n/,
-      /\A\/Permission/]
-
-    options[:fix_urls] = ->(url) do
-      return if url.include?('_') || url.include?('?')
-      url.sub! 'https://developer.mozilla.org/en-US/docs/DOM/', "#{Dom.base_url}/"
-      url.sub! 'https://developer.mozilla.org/en/DOM/',         "#{Dom.base_url}/"
-      url.sub! 'https://developer.mozilla.org/Web/API/',        "#{Dom.base_url}/"
-      url.sub! "#{Dom.base_url}/Console",                       "#{Dom.base_url}/console"
-      url.sub! "#{Dom.base_url}/Document\/",                    "#{Dom.base_url}/document\/"
-      url.sub! "#{Dom.base_url}/Element",                       "#{Dom.base_url}/element"
-      url.sub! "#{Dom.base_url}/History",                       "#{Dom.base_url}/history"
-      url.sub! "#{Dom.base_url}/Location",                      "#{Dom.base_url}/location"
-      url.sub! "#{Dom.base_url}/Navigator",                     "#{Dom.base_url}/navigator"
-      url.sub! "#{Dom.base_url}/Screen",                        "#{Dom.base_url}/screen"
-      url.sub! "#{Dom.base_url}/Window\/",                      "#{Dom.base_url}/window\/"
-      url.sub! "#{Dom.base_url}/notification",                  "#{Dom.base_url}/Notification"
-      url.sub! "#{Dom.base_url}/range",                         "#{Dom.base_url}/Range"
-      url.sub! "#{Dom.base_url}/event",                         "#{Dom.base_url}/Event"
-      url.sub! '/en/DOM/Manipulating_the_browser_history',      "/en-US/docs/Web/API/History_API"
-      url
-    end
   end
 end
diff --git a/lib/docs/scrapers/mdn/dom_events.rb b/lib/docs/scrapers/mdn/dom_events.rb
diff --git a/lib/docs/scrapers/mdn/html.rb b/lib/docs/scrapers/mdn/html.rb
@@ -7,8 +7,6 @@ class Html < Mdn
 
     html_filters.push 'html/clean_html', 'html/entries', 'title'
 
-    options[:mdn_tag] = 'HTML'
-
     options[:root_title] = 'HTML'
 
     options[:title] = ->(filter) do
diff --git a/lib/docs/scrapers/mdn/javascript.rb b/lib/docs/scrapers/mdn/javascript.rb
@@ -8,8 +8,6 @@ class Javascript < Mdn
 
     html_filters.push 'javascript/clean_html', 'javascript/entries', 'title'
 
-    options[:mdn_tag] = 'JavaScript'
-
     options[:root_title] = 'JavaScript'
 
     # Don't want
diff --git a/lib/docs/scrapers/mdn/mdn.rb b/lib/docs/scrapers/mdn/mdn.rb
@@ -3,33 +3,22 @@ class Mdn < UrlScraper
     self.abstract = true
     self.type = 'mdn'
 
-    params[:raw] = 1
-    params[:macros] = 1
-
     html_filters.push 'mdn/clean_html'
-    text_filters.insert_before 'attribution', 'mdn/contribute_link'
 
-    options[:rate_limit] = 200
+    options[:container] = '#content > .main-page-content'
     options[:trailing_slash] = false
 
     options[:skip_link] = ->(link) {
       link['title'].try(:include?, 'not yet been written'.freeze) && !link['href'].try(:include?, 'transform-function'.freeze)
     }
 
     options[:attribution] = <<-HTML
-      &copy; 2005&ndash;2020 Mozilla and individual contributors.<br>
+      &copy; 2005&ndash;2021 MDN contributors.<br>
       Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later.
     HTML
 
     def get_latest_version(opts)
       get_latest_github_commit_date('mdn', 'content', opts)
     end
-
-    private
-
-    def process_response?(response)
-      response.effective_url.host = 'developer.mozilla.org' if response.effective_url.host == 'wiki.developer.mozilla.org'
-      super && response.effective_url.query == 'raw=1&macros=1'
-    end
   end
 end
diff --git a/lib/docs/scrapers/mdn/svg.rb b/lib/docs/scrapers/mdn/svg.rb
@@ -8,8 +8,6 @@ class Svg < Mdn
 
     html_filters.push 'svg/clean_html', 'svg/entries', 'title'
 
-    options[:mdn_tag] = 'XSLT_Reference'
-
     options[:root_title] = 'SVG'
 
     options[:title] = ->(filter) do
diff --git a/lib/docs/scrapers/mdn/xslt_xpath.rb b/lib/docs/scrapers/mdn/xslt_xpath.rb
@@ -8,8 +8,6 @@ class XsltXpath < Mdn
 
     html_filters.push 'xslt_xpath/clean_html', 'xslt_xpath/entries', 'title'
 
-    options[:mdn_tag] = 'XSLT_Reference'
-
     options[:root_title] = 'XSLT'
 
     options[:only_patterns] = [/\A\/XSLT/, /\A\/XPath/]
