Fix bloomberg.com

* update UA for haaretz, themarker

* remove paywall for fortune.com

* [PATCH] fix themarker/haaretz user agent

README.md

@@ -26,16 +26,13 @@
 ### Bypass the following sites' paywalls with this extension:
 
 [Adweek](https://www.adweek.com)\
-[Algemeen Dagblad](https://www.ad.nl)\
 [American Banker](https://www.americanbanker.com)\
 [Ámbito](https://www.ambito.com)\
 [Baltimore Sun](https://www.baltimoresun.com)\
 [Barron's](https://www.barrons.com)\
 [Bloomberg Quint](https://www.bloombergquint.com)\
 [Bloomberg](https://www.bloomberg.com)\
-[BN De Stem](https://www.bndestem.nl)\
 [Boston Globe](https://www.bostonglobe.com)\
-[Brabants Dagblad](https://www.bd.nl)\
 [Brisbane Times](https://www.brisbanetimes.com.au)\
 [Business Insider](https://www.businessinsider.com)\
 [Caixin](https://www.caixinglobal.com)\
@@ -45,9 +42,7 @@
 [Corriere Della Sera](https://www.corriere.it)\
 [Crain's Chicago Business](https://www.chicagobusiness.com)\
 [Daily Press](https://www.dailypress.com)\
-[De Gelderlander](https://www.gelderlander.nl)\
 [De Groene Amsterdammer](https://www.groene.nl)\
-[De Stentor](https://www.destentor.nl)\
 [De Speld](https://speld.nl)\
 [De Tijd](https://www.tijd.be)\
 [De Volkskrant](https://www.volkskrant.nl)\
@@ -56,7 +51,6 @@
 [Diario Financiero](https://www.df.cl)\
 [Domani](https://www.editorialedomani.it)\
 [Dynamed Plus](https://www.dynamed.com)\
-[Eindhovens Dagblad](https://www.ed.nl)\
 [El Mercurio](https://www.elmercurio.com)\
 [El Pais](https://www.elpais.com)\
 [El Periodico](https://www.elperiodico.com)\
@@ -119,7 +113,6 @@
 [New Zealand Herald](https://www.nzherald.co.nz)\
 [Orange County Register](https://www.ocregister.com)\
 [Orlando Sentinel](https://www.orlandosentinel.com)\
-[PZC](https://www.pzc.nl)\
 [Palo Alto Online](https://www.paloaltoonline.com)\
 [Parool](https://www.parool.nl)\
 [Postimees](https://www.postimees.ee)\
@@ -188,7 +181,6 @@
 [Times Literary Supplement](https://www.the-tls.co.uk)\
 [Towards Data Science](https://www.towardsdatascience.com)\
 [Trouw](https://www.trouw.nl)\
-[Tubantia](https://www.tubantia.nl)\
 [Vanity Fair](https://www.vanityfair.com)\
 [Vrij Nederland](https://www.vn.nl)\
 [Vulture](https://www.vulture.com)\

manifest-ff.json

@@ -9,7 +9,6 @@
   "content_scripts": [
     {
     "matches": [
-    "*://*.ad.nl/*",
     "*://*.afr.com/*",
     "*://*.americanbanker.com/*",
     "*://*.ambito.com/*",
@@ -20,7 +19,6 @@
     "*://*.caixinglobal.com/*",
     "*://*.canberratimes.com.au/*",
     "*://*.economist.com/*",
-    "*://*.ed.nl/*",
     "*://*.editorialedomani.it/*",
     "*://*.elmercurio.com/*",
     "*://*.elpais.com/*",
@@ -125,7 +123,6 @@
     "*://*.poool.fr/*",
     "*://*.qiota.com/*",
     "*://*.tinypass.com/*",
-    "*://*.ad.nl/*",
     "*://*.adelaidenow.com.au/*",
     "*://*.adweek.com/*",
     "*://*.afr.com/*",
@@ -153,7 +150,6 @@
     "*://*.denverpost.com/*",
     "*://*.dynamed.com/*",
     "*://*.economist.com/*",
-    "*://*.ed.nl/*",
     "*://*.elmercurio.com/*",
     "*://*.elu24.ee/*",
     "*://*.examiner.com.au/*",
@@ -266,12 +262,6 @@
     "*://*.startribune.com/*",
     "*://*.df.cl/*",
     "*://*.financialpost.com/*",
-    "*://*.gelderlander.nl/*",
-    "*://*.bndestem.nl/*",
-    "*://*.bd.nl/*",
-    "*://*.tubantia.nl/*",
-    "*://*.pzc.nl/*",
-    "*://*.destentor.nl/*",
     "*://*.ambito/*",
     "*://*.sueddeutsche.de/*",
     "*://*.spectator.us/*",

src/js/background.js

@@ -6,24 +6,21 @@ const restrictions = {
   'economist.com': /.+economist\.com\/.+\/\d{1,4}\/\d{1,2}\/\d{2}\/.+/,
   'seekingalpha.com': /.+seekingalpha\.com\/article\/.+/,
   'techinasia.com': /\.techinasia\.com\/.+/,
-  'ft.com': /.+\.ft.com\/content\//
+  'ft.com': /.+\.ft.com\/content\//,
+  'nytimes.com': /^((?!\/timesmachine\.nytimes\.com\/).)*$/,
+  'theatlantic.com': /^((?!\/newsletters\.theatlantic\.com\/).)*$/,
+  'bloomberg.com': /^((?!\.bloomberg\.com\/news\/terminal\/).)*$/
 };
 
 // Don't remove cookies before page load
 const allowCookies = [
-  'ad.nl',
-  'bd.nl',
-  'bndestem.nl',
   'brisbanetimes.com.au',
   'canberratimes.com.au',
   'cen.acs.org',
   'demorgen.be',
   'denverpost.com',
-  'destentor.nl',
-  'ed.nl',
   'examiner.com.au',
   'gelocal.it',
-  'gelderlander.nl',
   'grubstreet.com',
   'harpers.org',
   'hbr.org',
@@ -37,7 +34,6 @@ const allowCookies = [
   'nymag.com',
   'ocregister.com',
   'parool.nl',
-  'pzc.nl',
   'qz.com',
   'scientificamerican.com',
   'seattletimes.com',
@@ -45,7 +41,6 @@ const allowCookies = [
   'sofrep.com',
   'spectator.co.uk',
   'speld.nl',
-  'tubantia.nl',
   'techinasia.com',
   'telegraaf.nl',
   'the-american-interest.com',
@@ -74,25 +69,20 @@ const allowCookies = [
   'zeit.de',
   'expansion.com',
   'dailytelegraph.com.au',
-  'washingtonpost.com'
+  'washingtonpost.com',
+  'nytimes.com'
 ];
 
 // Removes cookies after page load
 const removeCookies = [
-  'ad.nl',
-  'bd.nl',
   'bloomberg.com',
   'bloombergquint.com',
-  'bndestem.nl',
   'brisbanetimes.com.au',
   'canberratimes.com.au',
   'cen.acs.org',
   'demorgen.be',
   'denverpost.com',
-  'destentor.nl',
-  'ed.nl',
   'examiner.com.au',
-  'gelderlander.nl',
   'globes.co.il',
   'grubstreet.com',
   'harpers.org',
@@ -104,7 +94,6 @@ const removeCookies = [
   'nrc.nl',
   'nymag.com',
   'ocregister.com',
-  'pzc.nl',
   'qz.com',
   'scientificamerican.com',
   'seattletimes.com',
@@ -118,7 +107,6 @@ const removeCookies = [
   'thecut.com',
   'thediplomat.com',
   'towardsdatascience.com',
-  'tubantia.nl',
   'vn.nl',
   'vulture.com',
   'wsj.com',
@@ -139,19 +127,11 @@ const removeCookiesSelectHold = {
 
 // select only specific cookie(s) to drop from removeCookies domains
 const removeCookiesSelectDrop = {
-  'ad.nl': ['temptationTrackingId'],
   'ambito.com': ['TDNotesRead'],
-  'bd.nl': ['temptationTrackingId'],
-  'bndestem.nl': ['temptationTrackingId'],
   'demorgen.be': ['TID_ID'],
-  'destentor.nl': ['temptationTrackingId'],
-  'ed.nl': ['temptationTrackingId'],
   'fd.nl': ['socialread'],
-  'gelderlander.nl': ['temptationTrackingId'],
   'humo.be': ['TID_ID'],
   'nrc.nl': ['counter'],
-  'pzc.nl': ['temptationTrackingId'],
-  'tubantia.nl': ['temptationTrackingId'],
   'speld.nl': ['speld-paywall']
 };
 
@@ -178,11 +158,14 @@ const useGoogleBotSites = [
   'df.cl',
   'ft.com',
   'wired.com',
-  'zeit.de'
+  'zeit.de',
+  'nytimes.com'
 ];
 
 // Override User-Agent with Bingbot
-const useBingBot = [
+const useBingBot = [];
+
+const useMsnBot = [
   'haaretz.co.il',
   'haaretz.com',
   'themarker.com'
@@ -244,19 +227,22 @@ const blockedRegexes = {
   'chicagobusiness.com': /(\.tinypass\.com\/|\.chicagobusiness\.com\/.+\/js\/js_.+\.js)/,
   'dailytelegraph.com.au': /cdn\.ampproject\.org\/v\d\/amp-(access|ad|consent)-.+\.js/,
   'theglobeandmail.com': /(\.theglobeandmail\.com\/pf\/dist\/engine\/react\.js|smartwall\.theglobeandmail\.com\/)/,
-  'nytimes.com': /(meter-svc\.nytimes\.com\/meter\.js|mwcm\.nyt\.com\/.+\.js|cooking\.nytimes\.com\/api\/.+\/access)/,
-  'latimes.com': /(metering\.platform\.latimes\.com\/|cdn\.ampproject\.org\/v\d\/amp-(access|subscriptions)-.+\.js)/,
+  'nytimes.com': /(\.nytimes\.com\/meter\.js|mwcm\.nyt\.com\/.+\.js|cooking\.nytimes\.com\/api\/.+\/access)/,
+  'latimes.com': /\.californiatimes\.com\/meteringjs/,
   'theathletic.com': /cdn\.ampproject\.org\/v\d\/amp-(access|subscriptions)-.+\.js/,
   'japantimes.co.jp': /cdn\.cxense\.com\//,
   'scmp.com': /(\.tinypass\.com\/|cdn\.ampproject\.org\/v\d\/amp-access-.+\.js)/,
   'ilmessaggero.it': /(utils\.cedsdigital\.it\/js\/PaywallMeter\.js)/,
-  'washingtonpost.com': /\.washingtonpost\.com\/tetro\/metering\/evaluate/
+  'washingtonpost.com': /\.washingtonpost\.com\/tetro\/metering\/evaluate/,
+  'theatlantic.com': /cdn\.theatlantic\.com\/_next\/static\/chunks\/pages\/.+\/archive\//,
+  'bloomberg.com': /(\.cm\.bloomberg\.com\/|assets\.bwbx\.io\/s\d\/javelin\/.+\/transporter\/)/
 };
 
 const userAgentDesktop = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
 const userAgentMobile = 'Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)';
 const userAgentDesktopBingBot = 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)';
 const userAgentMobileBingBot = 'Chrome/80.0.3987.92 Mobile Safari/537.36 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)';
+const userAgentMsnBot = 'msnbot/2.0b (+http://search.msn.com/msnbot.htm)';
 
 let enabledSites = [];
 
@@ -325,6 +311,24 @@ extensionApi.webRequest.onBeforeRequest.addListener(function (details) {
 ['blocking']
 );
 
+// nytimes.com
+extensionApi.webRequest.onHeadersReceived.addListener(function (details) {
+  if (!isSiteEnabled(details)) {
+    return;
+  }
+  let headers = details.responseHeaders;
+  headers = headers.map(function (header) {
+    if (header.name === 'x-frame-options') { header.value = 'SAMEORIGIN'; }
+    return header;
+  });
+  return {
+    responseHeaders: headers
+  };
+}, {
+  urls: ['*://*.nytimes.com/*']
+},
+['blocking', 'responseHeaders']);
+
 // Disable javascript for these sites
 extensionApi.webRequest.onBeforeRequest.addListener(function (details) {
   const headerReferer = details.originUrl ? details.originUrl : details.initiator;
@@ -445,6 +449,14 @@ extensionApi.webRequest.onBeforeSendHeaders.addListener(function (details) {
     });
   }
 
+  // override User-Agent to use Google AdsBot Mobile Web
+  if (matchUrlDomain(useMsnBot, details.url)) {
+    requestHeaders.push({
+      name: 'User-Agent',
+      value: userAgentMsnBot
+    });
+  }
+
   // remove cookies before page load
   const enabledCookies = allowCookies.some(function (site) {
     return matchUrlDomain(site, details.url);
@@ -519,6 +531,24 @@ extensionApi.webRequest.onCompleted.addListener(function (details) {
   urls: ['<all_urls>']
 });
 
+// nytimes.com fix
+extensionApi.webRequest.onHeadersReceived.addListener(function (details) {
+  if (!isSiteEnabled(details)) {
+    return;
+  }
+  let headers = details.responseHeaders;
+  headers = headers.map(function (header) {
+    if (header.name === 'x-frame-options') { header.value = 'SAMEORIGIN'; }
+    return header;
+  });
+  return {
+    responseHeaders: headers
+  };
+}, {
+  urls: ['*://*.nytimes.com/*']
+},
+['blocking', 'responseHeaders']);
+
 // Google Analytics to anonymously track DAU (Chrome only)
 function initGA () {
   (function (i, s, o, g, r, a, m) {

src/js/contentScript.js

@@ -1,4 +1,4 @@
-if (!matchDomain(['seekingalpha.com', 'sfchronicle.com', 'cen.acs.org', 'elmundo.es', 'scmp.com'])) {
+if (!matchDomain(['seekingalpha.com', 'sfchronicle.com', 'cen.acs.org', 'elmundo.es', 'scmp.com', 'nytimes.com'])) {
   window.localStorage.clear();
 }
 
@@ -73,9 +73,6 @@ if (matchDomain('elmercurio.com')) {
       }
     }
   }
-} else if (matchDomain(['ad.nl', 'ed.nl', 'bndestem.nl', 'bd.nl', 'tubantia.nl', 'destentor.nl', 'pzc.nl', 'gelderlander.nl'])) {
-  const paywall = document.querySelector('.article__component.article__component--paywall-module-notification');
-  removeDOMElement(paywall);
 } else if (matchDomain('washingtonpost.com')) {
   const leaderboard = document.querySelector('#leaderboard-wrapper');
   const adverts = document.querySelectorAll('div[data-qa="article-body-ad"]');
@@ -207,7 +204,15 @@ if (matchDomain('elmercurio.com')) {
   const paywall = document.getElementsByClassName('paywall')[0];
   removeDOMElement(paywall);
 } else if (matchDomain('bloomberg.com')) {
-  blockElement('#graphics-paywall-overlay', true);
+  const paywall = document.querySelectorAll('div[id^="fortress-"]');
+  const leaderboard = document.querySelector('div[id^="leaderboard"], div[class^="leaderboard"], div.canopy-container');
+  const ads = document.querySelectorAll('div[data-ad-status], div.dvz-v0-ad, div[class^="FullWidthAd_"]');
+  hideDOMElement(...paywall, leaderboard, ...ads);
+  if (window.location.pathname.startsWith('/live/')) {
+    setInterval(function () {
+      window.localStorage.clear();
+    }, 15 * 60 * 1000);
+  }
 } else if (matchDomain('bloombergquint.com')) {
   const articlesLeftModal = document.getElementsByClassName('paywall-meter-module__story-paywall-container__1UgCE')[0];
   const paywall = document.getElementById('paywallDmp');
@@ -261,7 +266,7 @@ if (matchDomain('elmercurio.com')) {
     removeDOMElement(counter, coBanner, support);
   });
 } else if (matchDomain('nytimes.com')) {
-  const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div.expanded-dock');
+  const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div.expanded-dock, div.pz-ad-box');
   removeDOMElement(...banners);
 } else if (matchDomain('technologyreview.com')) {
   window.setTimeout(function () {
@@ -449,14 +454,10 @@ if (matchDomain('elmercurio.com')) {
       });
   }, 1000); // Delay (in milliseconds)
 } else if (matchDomain('theatlantic.com')) {
-  // Remove all nudge elements
-  document.querySelectorAll('div[class*="c-nudge"]').forEach(function (el) {
-    removeDOMElement(el);
-  });
-  // Remove all FancyBox ads
-  document.querySelectorAll('div[class*="fancybox"]').forEach(function (el) {
-    removeDOMElement(el);
-  });
+  const images = document.querySelectorAll('img[class*="Image_lazy__"]');
+  for (const elem of images) { removeClassesByPrefix(elem, 'Image_lazy__'); }
+  const banners = document.querySelectorAll('.c-nudge__container, .c-non-metered-nudge, div[class^="ArticleInjector_"]');
+  hideDOMElement(...banners);
 } else if (matchDomain('theathletic.com')) {
   if (!window.location.search.match(/(\?|&)amp/)) {
     const paywall = document.querySelector('div#slideup-paywall');
@@ -644,16 +645,8 @@ if (matchDomain('elmercurio.com')) {
     body.removeAttribute('class');
   }
 } else if (matchDomain('latimes.com')) {
-  const paywall = document.querySelector('metering-modal');
-  const incognitoWall = document.querySelector('metering-toppanel');
-  if (paywall) {
-    removeDOMElement(paywall);
-  } else if (incognitoWall) {
-    removeDOMElement(incognitoWall);
-  }
-  if (paywall || incognitoWall) {
-    document.body.removeAttribute('style');
-  }
+  const ads = document.querySelectorAll('div.enhancement, div.google-dfp-ad-wrapper');
+  hideDOMElement(...ads);
 } else if (matchDomain('foreignpolicy.com')) {
   const contentUngated = document.querySelector('div.content-ungated');
   removeDOMElement(contentUngated);
@@ -716,6 +709,13 @@ if (matchDomain('elmercurio.com')) {
     const ads = document.querySelectorAll('amp-ad, div.ad-banner, div.advert-fly-carpet-container, div.inline-advert');
     removeDOMElement(...defaultMeters, ...ads);
   }
+} else if (matchDomain('fortune.com')) {
+  const paywalledArticle = document.querySelector('.paywall.paywallActive');
+  if (paywalledArticle) {
+    for (const clazz of ['paywall', 'paywallActive']) {
+      paywalledArticle.classList.remove(clazz);
+    }
+  }
 }
 
 function matchDomain (domains) {

src/js/sites.js

@@ -1,16 +1,13 @@
 // Cookies from this list are blocked by default
 const defaultSites = {
   'Adweek': 'adweek.com',
-  'Algemeen Dagblad': 'ad.nl',
   '\u00C1mbito': 'ambito.com',
   'American Banker': 'americanbanker.com',
   'Baltimore Sun': 'baltimoresun.com',
   'Barron\'s': 'barrons.com',
   'Bloomberg': 'bloomberg.com',
   'Bloomberg Quint (free articles only)': 'bloombergquint.com',
-  'BN De Stem': 'bndestem.nl',
   'Boston Globe': 'bostonglobe.com',
-  'Brabants Dagblad': 'bd.nl',
   'Brisbane Times': 'brisbanetimes.com.au',
   'Business Insider': 'businessinsider.com',
   'Caixin': 'caixinglobal.com',
@@ -20,18 +17,15 @@ const defaultSites = {
   'Corriere Della Sera': 'corriere.it',
   'Crain\'s Chicago Business': 'chicagobusiness.com',
   'Daily Press': 'dailypress.com',
-  'De Gelderlander': 'gelderlander.nl',
   'De Groene Amsterdammer': 'groene.nl',
   'DeMorgen': 'demorgen.be',
   'Denver Post': 'denverpost.com',
   'De Speld': 'speld.nl',
-  'De Stentor': 'destentor.nl',
   'De Tijd': 'tijd.be',
   'de Volkskrant': 'volkskrant.nl',
   'Diario Financiero': 'df.cl',
   'Domani': 'editorialedomani.it',
   'Dynamed Plus': 'dynamed.com',
-  'Eindhovens Dagblad': 'ed.nl',
   'El Mercurio': 'elmercurio.com',
   'El Mundo': 'elmundo.es',
   'El Pais': 'elpais.com',
@@ -99,7 +93,6 @@ const defaultSites = {
   'Palo Alto Online': 'paloaltoonline.com',
   'Parool': 'parool.nl',
   'Postimees': 'postimees.ee',
-  'PZC': 'pzc.nl',
   'Quartz (free articles only)': 'qz.com',
   'Quora': 'quora.com',
   'Quotidiani Gelocal': 'gelocal.it',
@@ -168,7 +161,6 @@ const defaultSites = {
   'Times Literary Supplement': 'the-tls.co.uk',
   'Towards Data Science': 'towardsdatascience.com',
   'Trouw': 'trouw.nl',
-  'Tubantia': 'tubantia.nl',
   'Vanity Fair': 'vanityfair.com',
   'Vrij Nederland': 'vn.nl',
   'Vulture': 'vulture.com',