From 5d964aa60e7305af7eba7e88c0c6dafaad39fb16 Mon Sep 17 00:00:00 2001
From: fkrebs ⏳ Paywall detected — will retry archive.ph 🚫 Paywall — gave up after 3h 🔓 Open access via Unpaywall 📰 Free article — full text via crawl4ai 📖 Full text via archive.ph 📖 Full text via crawl4ai (not OA) 📖 Paywall bypass via crawl4ai
' + archiveLink + '
' + archiveLink + '');\n s = s.replace(/\\[([^\\]]+)\\]\\(([^)\\s]+)[^)]*\\)/g, '$1');\n s = s.replace(/\\*\\*([^*\\n]+)\\*\\*/g, '$1');\n s = s.replace(/`([^`\\n]+)`/g, '
$1');\n const out = [];\n let inList = false, inPara = false;\n for (const raw of s.split('\\n')) {\n const line = raw.trim();\n if (!line) {\n if (inList) { out.push(''); inList = false; }\n if (inPara) { out.push('
'); inPara = true; }\n out.push(line);\n }\n if (inList) out.push('');\n if (inPara) out.push('
');\n return out.join('\\n');\n}\n\nconst self = this;\nasync function fetchCrawl4ai(url) {\n try {\n const cr = await self.helpers.httpRequest({\n method: 'POST',\n url: 'http://mcp-crawl4ai:11235/md',\n headers: { 'Content-Type': 'application/json' },\n body: { url, filter: 'fit' },\n json: true,\n timeout: 60000,\n });\n if (cr && cr.success && cr.markdown) return cr.markdown;\n } catch (_) {}\n return '';\n}\n\nasync function fetchUnpaywall(doi) {\n try {\n const up = await self.helpers.httpRequest({\n method: 'GET',\n url: `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=fkrebs@nucli.de`,\n json: true,\n timeout: 15000,\n });\n if (up.is_oa && up.best_oa_location) {\n return up.best_oa_location.url || up.best_oa_location.url_for_pdf || '';\n }\n } catch (_) {}\n return '';\n}\n\nasync function getArchiveSnapshotUrl(url) {\n try {\n const tm = await self.helpers.httpRequest({\n method: 'GET',\n url: 'https://archive.ph/timemap/' + url,\n headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0' },\n timeout: 30000,\n });\n const txt = typeof tm === 'string' ? tm : (tm && tm.body) || '';\n const matches = [...txt.matchAll(/<([^>]+)>;\\s*rel=\"[^\"]*memento[^\"]*\"/g)];\n if (matches.length > 0) return matches[matches.length - 1][1];\n } catch (_) {}\n return '';\n}\n\nconst entries = $('Get Miniflux Entries').first().json.entries || [];\nconst results = [];\nconst stats = { skipped_done: 0, skipped_nopaywall: 0, attempted: 0, success: 0, retry: 0, giveup: 0, error: 0 };\n\nfor (const e of entries) {\n if (ALREADY_PROCESSED.test(e.title)) { stats.skipped_done++; continue; }\n\n const host = hostname(e.url);\n const isScience = PAYWALL_SCIENCE.some(d => host === d || host.endsWith('.' + d));\n const isNews = PAYWALL_NEWS.some(d => host === d || host.endsWith('.' + d));\n const isShort = (e.content || '').replace(/<[^>]+>/g, '').trim().length < 800;\n\n if (!isScience && !isNews && !isShort) { stats.skipped_nopaywall++; continue; }\n stats.attempted++;\n\n const type = isScience ? 'science' : isNews ? 'news' : 'short';\n const publishedMs = new Date(e.published_at).getTime();\n const ageMs = Date.now() - publishedMs;\n const exhausted = ageMs >= RETRY_LIMIT_MS;\n\n let markdown = '';\n let source = '';\n\n if (isScience) {\n const doi = extractDoi(e.url);\n if (doi) {\n const oaUrl = await fetchUnpaywall(doi);\n if (oaUrl) {\n const md = await fetchCrawl4ai(oaUrl);\n if (md && !isPaywalled(md)) { markdown = md; source = 'unpaywall'; }\n }\n }\n }\n\n // Try crawl4ai direct first to find out if it's actually paywalled\n if (!source) {\n const md = await fetchCrawl4ai(e.url);\n if (md) {\n if (!isPaywalled(md)) {\n markdown = md;\n source = 'free';\n } else {\n markdown = md; // keep for retry/giveup display\n }\n }\n }\n\n // If paywalled (or no content), try archive.ph for bypass\n if (!source) {\n const snapUrl = await getArchiveSnapshotUrl(e.url);\n if (snapUrl) {\n const md = await fetchCrawl4ai(snapUrl);\n if (md && !isPaywalled(md)) { markdown = md; source = 'archive'; }\n }\n }\n\n const cleanTitle = e.title.replace(ANY_STATUS, '');\n let newTitle, content, status;\n\n if (source) {\n content = badge(source, type, 'ok') + mdToHtml(markdown);\n const emoji = source === 'unpaywall' ? '🔓' : source === 'free' ? '📰' : '📖';\n newTitle = cleanTitle + ' ' + emoji;\n stats.success++;\n status = 'updated';\n } else if (markdown && exhausted) {\n content = badge('', type, 'giveup', e.url) + mdToHtml(markdown);\n newTitle = cleanTitle + ' 🚫';\n stats.giveup++;\n status = 'gaveup';\n } else if (markdown) {\n content = badge('', type, 'paywalled', e.url) + mdToHtml(markdown);\n newTitle = cleanTitle + ' ⏳';\n stats.retry++;\n status = 'retry';\n } else {\n content = '❌ Fetch error
Check archive.ph manually →
\u23f3 Paywall detected \u2014 will retry archive.ph
' + archiveLink + '
\ud83d\udeab Paywall \u2014 gave up after 3h
' + archiveLink + '
\ud83d\udd13 Open access via Unpaywall
';\n if (source === 'free') return '\ud83d\udcf0 Free article \u2014 full text via crawl4ai
';\n if (source === 'archive') return '\ud83d\udcd6 Full text via archive.ph
';\n if (type === 'science') return '\ud83d\udcd6 Full text via crawl4ai (not OA)
';\n return '\ud83d\udcd6 Paywall bypass via crawl4ai
';\n}\n\nfunction mdToHtml(md) {\n if (!md) return '';\n const lines = md.split('\\n').filter(l => {\n const t = l.trim();\n if (!t) return true;\n if (/Zur Merkliste hinzuf\u00fcgen|Artikel anh\u00f6ren|Bild vergr\u00f6\u00dfern|Bild schlie\u00dfen|Link kopieren|Weitere Optionen zum Teilen|Mehr lesen \u00fcber|Verwandte Artikel/.test(t)) return false;\n if (/^\\s*\\*\\s*\\[?\\s*(X\\.com|Facebook|Messenger|WhatsApp|E-Mail|Threads|Mastodon|Telegram)\\b/i.test(t)) return false;\n return true;\n });\n let s = lines.join('\\n');\n s = s.replace(/!\\[([^\\]]*)\\]\\(([^)\\s]+)[^)]*\\)/g, '$1');\n const out = [];\n let inList = false, inPara = false;\n for (const raw of s.split('\\n')) {\n const line = raw.trim();\n if (!line) {\n if (inList) { out.push(''); inList = false; }\n if (inPara) { out.push(''); inPara = false; }\n continue;\n }\n const h = line.match(/^(#{1,6})\\s+(.+)$/);\n if (h) {\n if (inList) { out.push(''); inList = false; }\n if (inPara) { out.push(''); inPara = false; }\n const n = h[1].length;\n out.push(`'); inPara = true; }\n out.push(line);\n }\n if (inList) out.push('');\n if (inPara) out.push('
');\n return out.join('\\n');\n}\n\nconst self = this;\nasync function fetchCrawl4ai(url) {\n try {\n const cr = await self.helpers.httpRequest({\n method: 'POST',\n url: 'http://mcp-crawl4ai:11235/md',\n headers: { 'Content-Type': 'application/json' },\n body: { url, filter: 'fit' },\n json: true,\n timeout: 60000,\n });\n if (cr && cr.success && cr.markdown) return cr.markdown;\n } catch (_) {}\n return '';\n}\n\nasync function fetchUnpaywall(doi) {\n try {\n const up = await self.helpers.httpRequest({\n method: 'GET',\n url: `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=fkrebs@nucli.de`,\n json: true,\n timeout: 15000,\n });\n if (up.is_oa && up.best_oa_location) {\n return up.best_oa_location.url || up.best_oa_location.url_for_pdf || '';\n }\n } catch (_) {}\n return '';\n}\n\nasync function getArchiveSnapshotUrl(url) {\n try {\n const tm = await self.helpers.httpRequest({\n method: 'GET',\n url: 'https://archive.ph/timemap/' + url,\n headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0' },\n timeout: 30000,\n });\n const txt = typeof tm === 'string' ? tm : (tm && tm.body) || '';\n const matches = [...txt.matchAll(/<([^>]+)>;\\s*rel=\"[^\"]*memento[^\"]*\"/g)];\n if (matches.length > 0) return matches[matches.length - 1][1];\n } catch (_) {}\n return '';\n}\n\nconst entries = $('Get Miniflux Entries').first().json.entries || [];\nconst results = [];\nconst stats = { skipped_done: 0, skipped_nopaywall: 0, attempted: 0, success: 0, retry: 0, giveup: 0, error: 0 };\n\nfor (const e of entries) {\n if (ALREADY_PROCESSED.test(e.title)) { stats.skipped_done++; continue; }\n\n const host = hostname(e.url);\n const isScience = PAYWALL_SCIENCE.some(d => host === d || host.endsWith('.' + d));\n const isNews = PAYWALL_NEWS.some(d => host === d || host.endsWith('.' + d));\n const isShort = (e.content || '').replace(/<[^>]+>/g, '').trim().length < 800;\n\n if (!isScience && !isNews && !isShort) { stats.skipped_nopaywall++; continue; }\n stats.attempted++;\n\n const type = isScience ? 'science' : isNews ? 'news' : 'short';\n const publishedMs = new Date(e.published_at).getTime();\n const ageMs = Date.now() - publishedMs;\n const exhausted = ageMs >= RETRY_LIMIT_MS;\n\n let markdown = '';\n let source = '';\n\n if (isScience) {\n const doi = extractDoi(e.url);\n if (doi) {\n const oaUrl = await fetchUnpaywall(doi);\n if (oaUrl) {\n const md = await fetchCrawl4ai(oaUrl);\n if (md && !isPaywalled(md)) { markdown = md; source = 'unpaywall'; }\n }\n }\n }\n\n // Try crawl4ai direct first to find out if it's actually paywalled\n if (!source) {\n const md = await fetchCrawl4ai(e.url);\n if (md) {\n if (!isPaywalled(md)) {\n markdown = md;\n source = 'free';\n } else {\n markdown = md; // keep for retry/giveup display\n }\n }\n }\n\n // If paywalled (or no content), try archive.ph for bypass\n if (!source) {\n const snapUrl = await getArchiveSnapshotUrl(e.url);\n if (snapUrl) {\n const md = await fetchCrawl4ai(snapUrl);\n if (md && !isPaywalled(md)) { markdown = md; source = 'archive'; }\n }\n }\n\n const cleanTitle = e.title.replace(ANY_STATUS, '');\n let newTitle, content, status;\n\n if (source) {\n content = badge(source, type, 'ok') + mdToHtml(markdown);\n const emoji = source === 'unpaywall' ? '\ud83d\udd13' : source === 'free' ? '\ud83d\udcf0' : '\ud83d\udcd6';\n newTitle = cleanTitle + ' ' + emoji;\n stats.success++;\n status = 'updated';\n } else if (markdown && exhausted) {\n content = badge('', type, 'giveup', e.url) + mdToHtml(markdown);\n newTitle = cleanTitle + ' \ud83d\udeab';\n stats.giveup++;\n status = 'gaveup';\n } else if (markdown) {\n content = badge('', type, 'paywalled', e.url) + mdToHtml(markdown);\n newTitle = cleanTitle + ' \u23f3';\n stats.retry++;\n status = 'retry';\n } else {\n content = '\u274c Fetch error
Check archive.ph manually \u2192