From d9489095bf17de94e06d6ccc5ff3ca8774489842 Mon Sep 17 00:00:00 2001 From: fkrebs Date: Thu, 28 May 2026 07:13:33 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20archive.ph=20fallback=20+=20paywall=20d?= =?UTF-8?q?etection=20+=203h=20retry/giveup=20+=20markdown=E2=86=92HTML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - archive.ph timemap → snapshot URL → crawl4ai fetch (low rate-limit endpoint) - Unpaywall OA lookup for DOIs (science) - crawl4ai direct as final fallback - Detect paywall markers (Spiegel S+, generic abonnieren) + content length - Status icons: 🔓 OA · 📖 fetched · ⏳ retry · 🚫 gave up (3h) · ❌ error - mdToHtml inline converter (lists/links/headings/bold/code) + strips nav/share lines - regex hostname extractor (URL constructor undefined in n8n task runner sandbox) --- flows/paywall-bypass.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flows/paywall-bypass.json b/flows/paywall-bypass.json index 62c8b0c..0985ceb 100644 --- a/flows/paywall-bypass.json +++ b/flows/paywall-bypass.json @@ -82,7 +82,7 @@ ], "parameters": { "mode": "runOnceForAllItems", - "jsCode": "const MINIFLUX_TOKEN = '3f439f2db55967ed5acd83398214d2aa76124927cde07115d945b9282597fecb';\nconst MINIFLUX_URL = 'http://192.168.1.40:17002';\n\nconst PAYWALL_NEWS = [\n 'spiegel.de','zeit.de','faz.net','sueddeutsche.de','handelsblatt.com',\n 'nytimes.com','wsj.com','ft.com','washingtonpost.com','thetimes.co.uk',\n 'wired.com','theatlantic.com','economist.com','bloomberg.com',\n 'hbr.org','foreignpolicy.com','newyorker.com','taz.de',\n 'telegraph.co.uk','businessinsider.com'\n];\n\nconst PAYWALL_SCIENCE = [\n 'nature.com','science.org','cell.com','nejm.org','thelancet.com',\n 'link.springer.com','springer.com','wiley.com','onlinelibrary.wiley.com',\n 'sciencedirect.com','pubs.acs.org','rsc.org','pubs.rsc.org',\n 'bmj.com','annualreviews.org','pnas.org',\n 'jamanetwork.com','ahajournals.org','tandfonline.com','academic.oup.com'\n];\n\nconst CONTENT_MIN = 800;\nconst ALREADY_PROCESSED = /[\\u{1F513}\\u{1F4D6}❌]$/u;\n\nfunction extractDoi(url) {\n let m = url.match(/doi\\.org\\/(10\\.\\d{4,}\\/[^\\s?#]+)/);\n if (m) return decodeURIComponent(m[1]);\n m = url.match(/\\/doi\\/(10\\.\\d{4,}\\/[^\\s?#]+)/);\n if (m) return decodeURIComponent(m[1].replace(/[?#].*$/, ''));\n return null;\n}\n\nfunction hostname(url) {\n try { return new URL(url).hostname.replace(/^www\\./, ''); } catch { return ''; }\n}\n\nfunction badge(source, type) {\n if (source === 'unpaywall') return '

🔓 Open access via Unpaywall

';\n if (type === 'science') return '

📖 Full text via crawl4ai (not OA)

';\n return '

📖 Paywall bypass via crawl4ai

';\n}\n\nconst entries = $('Get Miniflux Entries').first().json.entries || [];\nconst results = [];\nconst stats = { skipped_processed: 0, skipped_nopaywall: 0, fetched: 0, failed: 0, updated: 0 };\n\nfor (const e of entries) {\n if (ALREADY_PROCESSED.test(e.title)) { stats.skipped_processed++; continue; }\n\n const host = hostname(e.url);\n const isScience = PAYWALL_SCIENCE.some(d => host === d || host.endsWith('.' + d));\n const isNews = PAYWALL_NEWS.some(d => host === d || host.endsWith('.' + d));\n const isShort = (e.content || '').replace(/<[^>]+>/g, '').trim().length < CONTENT_MIN;\n\n if (!isScience && !isNews && !isShort) { stats.skipped_nopaywall++; continue; }\n\n const type = isScience ? 'science' : isNews ? 'news' : 'short';\n let fetchUrl = e.url;\n let source = 'crawl4ai';\n\n if (isScience) {\n const doi = extractDoi(e.url);\n if (doi) {\n try {\n const up = await this.helpers.httpRequest({\n method: 'GET',\n url: `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=fkrebs@nucli.de`,\n json: true,\n });\n if (up.is_oa && up.best_oa_location) {\n const oaUrl = up.best_oa_location.url || up.best_oa_location.url_for_pdf;\n if (oaUrl) { fetchUrl = oaUrl; source = 'unpaywall'; }\n }\n } catch (_) {}\n }\n }\n\n let fullContent = '';\n let crawlErr = '';\n try {\n const cr = await this.helpers.httpRequest({\n method: 'POST',\n url: 'http://mcp-crawl4ai:11235/md',\n headers: { 'Content-Type': 'application/json' },\n body: { url: fetchUrl, filter: 'fit' },\n json: true,\n });\n if (cr && cr.success && cr.markdown) {\n fullContent = '

' + cr.markdown.replace(/\\n\\n/g, '

').replace(/\\n/g, '
') + '

';\n stats.fetched++;\n } else {\n crawlErr = 'no_markdown';\n }\n } catch (err) {\n crawlErr = String(err).slice(0, 200);\n }\n\n if (!fullContent) {\n stats.failed++;\n const failTitle = e.title.replace(/\\s[\\u{1F513}\\u{1F4D6}❌]$/u, '') + ' ❌';\n try {\n await this.helpers.httpRequest({\n method: 'PUT',\n url: `${MINIFLUX_URL}/v1/entries/${e.id}`,\n headers: { 'X-Auth-Token': MINIFLUX_TOKEN, 'Content-Type': 'application/json' },\n body: { title: failTitle },\n json: true,\n });\n } catch (_) {}\n results.push({ json: { entry_id: e.id, title: failTitle, status: 'no_content', source, type, error: crawlErr } });\n continue;\n }\n\n const finalContent = badge(source, type) + fullContent;\n const titleEmoji = source === 'unpaywall' ? '🔓' : '📖';\n const newTitle = e.title.replace(/\\s[\\u{1F513}\\u{1F4D6}❌]$/u, '') + ' ' + titleEmoji;\n\n try {\n await this.helpers.httpRequest({\n method: 'PUT',\n url: `${MINIFLUX_URL}/v1/entries/${e.id}`,\n headers: { 'X-Auth-Token': MINIFLUX_TOKEN, 'Content-Type': 'application/json' },\n body: { content: finalContent, title: newTitle },\n json: true,\n });\n stats.updated++;\n results.push({ json: { entry_id: e.id, title: newTitle, status: 'updated', source, type } });\n } catch (err) {\n results.push({ json: { entry_id: e.id, title: e.title, status: 'update_error', error: String(err).slice(0, 200), source, type } });\n }\n}\n\nresults.push({ json: { stats } });\nreturn results;" + "jsCode": "const MINIFLUX_TOKEN = '3f439f2db55967ed5acd83398214d2aa76124927cde07115d945b9282597fecb';\nconst MINIFLUX_URL = 'http://192.168.1.40:17002';\n\nconst PAYWALL_NEWS = [\n 'spiegel.de','zeit.de','faz.net','sueddeutsche.de','handelsblatt.com',\n 'nytimes.com','wsj.com','ft.com','washingtonpost.com','thetimes.co.uk',\n 'wired.com','theatlantic.com','economist.com','bloomberg.com',\n 'hbr.org','foreignpolicy.com','newyorker.com','taz.de',\n 'telegraph.co.uk','businessinsider.com'\n];\nconst PAYWALL_SCIENCE = [\n 'nature.com','science.org','cell.com','nejm.org','thelancet.com',\n 'link.springer.com','springer.com','wiley.com','onlinelibrary.wiley.com',\n 'sciencedirect.com','pubs.acs.org','rsc.org','pubs.rsc.org',\n 'bmj.com','annualreviews.org','pnas.org',\n 'jamanetwork.com','ahajournals.org','tandfonline.com','academic.oup.com'\n];\n\nconst CONTENT_MIN = 1500;\nconst RETRY_LIMIT_MS = 3 * 60 * 60 * 1000;\n\nconst ALREADY_PROCESSED = /[\\u{1F513}\\u{1F4D6}\\u{1F6AB}\\u{274C}]$/u;\nconst ANY_STATUS = /\\s[\\u{1F513}\\u{1F4D6}\\u{1F6AB}\\u{274C}⏳]$/u;\n\nconst PAYWALL_MARKERS = [\n 'Sie können den Artikel leider nicht mehr aufrufen',\n 'Diesen Artikel weiterlesen mit',\n 'SPIEGEL plus',\n 'Jetzt abonnieren',\n 'Bereits Abonnent',\n 'Subscribe to read',\n 'Become a subscriber',\n 'To continue reading',\n 'Get full access',\n 'um diesen Artikel zu lesen',\n 'Mit Digital-Abo',\n];\n\nfunction hostname(url) {\n const m = String(url || '').match(/^[a-z]+:\\/\\/([^/?#:]+)/i);\n if (!m) return '';\n return m[1].toLowerCase().replace(/^www\\./, '');\n}\n\nfunction isPaywalled(text) {\n if (!text) return true;\n const clean = text.replace(/<[^>]+>/g, '').trim();\n if (clean.length < CONTENT_MIN) return true;\n return PAYWALL_MARKERS.some(m => text.includes(m));\n}\n\nfunction extractDoi(url) {\n let m = url.match(/doi\\.org\\/(10\\.\\d{4,}\\/[^\\s?#]+)/);\n if (m) return decodeURIComponent(m[1]);\n m = url.match(/\\/doi\\/(10\\.\\d{4,}\\/[^\\s?#]+)/);\n if (m) return decodeURIComponent(m[1].replace(/[?#].*$/, ''));\n return null;\n}\n\nfunction badge(source, type, status) {\n if (status === 'paywalled') return '

⏳ Paywall detected — will retry archive.ph (latest fetch shown below)

';\n if (status === 'giveup') return '

🚫 Paywall — gave up after 3h (latest attempt below)

';\n if (source === 'unpaywall') return '

🔓 Open access via Unpaywall

';\n if (source === 'archive') return '

📖 Full text via archive.ph

';\n if (type === 'science') return '

📖 Full text via crawl4ai (not OA)

';\n return '

📖 Paywall bypass via crawl4ai

';\n}\n\nfunction mdToHtml(md) {\n if (!md) return '';\n const lines = md.split('\\n').filter(l => {\n const t = l.trim();\n if (!t) return true;\n if (/Zur Merkliste hinzufügen|Artikel anhören|Bild vergrößern|Bild schließen|Link kopieren|Weitere Optionen zum Teilen|Mehr lesen über|Verwandte Artikel/.test(t)) return false;\n if (/^\\s*\\*\\s*\\[?\\s*(X\\.com|Facebook|Messenger|WhatsApp|E-Mail|Threads|Mastodon|Telegram)\\b/i.test(t)) return false;\n return true;\n });\n let s = lines.join('\\n');\n s = s.replace(/!\\[([^\\]]*)\\]\\(([^)\\s]+)[^)]*\\)/g, '\"$1\"');\n s = s.replace(/\\[([^\\]]+)\\]\\(([^)\\s]+)[^)]*\\)/g, '$1');\n s = s.replace(/\\*\\*([^*\\n]+)\\*\\*/g, '$1');\n s = s.replace(/`([^`\\n]+)`/g, '$1');\n const out = [];\n let inList = false, inPara = false;\n for (const raw of s.split('\\n')) {\n const line = raw.trim();\n if (!line) {\n if (inList) { out.push(''); inList = false; }\n if (inPara) { out.push('

'); inPara = false; }\n continue;\n }\n const h = line.match(/^(#{1,6})\\s+(.+)$/);\n if (h) {\n if (inList) { out.push(''); inList = false; }\n if (inPara) { out.push('

'); inPara = false; }\n const n = h[1].length;\n out.push(`${h[2]}`);\n continue;\n }\n const li = line.match(/^[*\\-]\\s+(.+)$/);\n if (li) {\n if (inPara) { out.push('

'); inPara = false; }\n if (!inList) { out.push(''); inList = false; }\n if (!inPara) { out.push('

'); inPara = true; }\n out.push(line);\n }\n if (inList) out.push('');\n if (inPara) out.push('

');\n return out.join('\\n');\n}\n\nconst self = this;\nasync function fetchCrawl4ai(url) {\n try {\n const cr = await self.helpers.httpRequest({\n method: 'POST',\n url: 'http://mcp-crawl4ai:11235/md',\n headers: { 'Content-Type': 'application/json' },\n body: { url, filter: 'fit' },\n json: true,\n timeout: 60000,\n });\n if (cr && cr.success && cr.markdown) return cr.markdown;\n } catch (_) {}\n return '';\n}\n\nasync function fetchUnpaywall(doi) {\n try {\n const up = await self.helpers.httpRequest({\n method: 'GET',\n url: `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=fkrebs@nucli.de`,\n json: true,\n timeout: 15000,\n });\n if (up.is_oa && up.best_oa_location) {\n return up.best_oa_location.url || up.best_oa_location.url_for_pdf || '';\n }\n } catch (_) {}\n return '';\n}\n\nasync function getArchiveSnapshotUrl(url) {\n try {\n const tm = await self.helpers.httpRequest({\n method: 'GET',\n url: 'https://archive.ph/timemap/' + url,\n headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0' },\n timeout: 30000,\n });\n const txt = typeof tm === 'string' ? tm : (tm && tm.body) || '';\n const matches = [...txt.matchAll(/<([^>]+)>;\\s*rel=\"[^\"]*memento[^\"]*\"/g)];\n if (matches.length > 0) return matches[matches.length - 1][1];\n } catch (_) {}\n return '';\n}\n\nconst entries = $('Get Miniflux Entries').first().json.entries || [];\nconst results = [];\nconst stats = { skipped_done: 0, skipped_nopaywall: 0, attempted: 0, success: 0, retry: 0, giveup: 0, error: 0 };\n\nfor (const e of entries) {\n if (ALREADY_PROCESSED.test(e.title)) { stats.skipped_done++; continue; }\n\n const host = hostname(e.url);\n const isScience = PAYWALL_SCIENCE.some(d => host === d || host.endsWith('.' + d));\n const isNews = PAYWALL_NEWS.some(d => host === d || host.endsWith('.' + d));\n const isShort = (e.content || '').replace(/<[^>]+>/g, '').trim().length < 800;\n\n if (!isScience && !isNews && !isShort) { stats.skipped_nopaywall++; continue; }\n stats.attempted++;\n\n const type = isScience ? 'science' : isNews ? 'news' : 'short';\n const publishedMs = new Date(e.published_at).getTime();\n const ageMs = Date.now() - publishedMs;\n const exhausted = ageMs >= RETRY_LIMIT_MS;\n\n let markdown = '';\n let source = '';\n\n if (isScience) {\n const doi = extractDoi(e.url);\n if (doi) {\n const oaUrl = await fetchUnpaywall(doi);\n if (oaUrl) {\n const md = await fetchCrawl4ai(oaUrl);\n if (md && !isPaywalled(md)) { markdown = md; source = 'unpaywall'; }\n }\n }\n }\n\n if (!source) {\n const snapUrl = await getArchiveSnapshotUrl(e.url);\n if (snapUrl) {\n const md = await fetchCrawl4ai(snapUrl);\n if (md && !isPaywalled(md)) { markdown = md; source = 'archive'; }\n }\n }\n\n if (!source) {\n const md = await fetchCrawl4ai(e.url);\n if (md && !isPaywalled(md)) { markdown = md; source = 'crawl4ai'; }\n else if (md && !markdown) markdown = md;\n }\n\n const cleanTitle = e.title.replace(ANY_STATUS, '');\n let newTitle, content, status;\n\n if (source) {\n content = badge(source, type, 'ok') + mdToHtml(markdown);\n const emoji = source === 'unpaywall' ? '🔓' : '📖';\n newTitle = cleanTitle + ' ' + emoji;\n stats.success++;\n status = 'updated';\n } else if (markdown && exhausted) {\n content = badge('', type, 'giveup') + mdToHtml(markdown);\n newTitle = cleanTitle + ' 🚫';\n stats.giveup++;\n status = 'gaveup';\n } else if (markdown) {\n content = badge('', type, 'paywalled') + mdToHtml(markdown);\n newTitle = cleanTitle + ' ⏳';\n stats.retry++;\n status = 'retry';\n } else {\n content = null;\n newTitle = cleanTitle + ' ❌';\n stats.error++;\n status = 'error';\n }\n\n const body = content !== null ? { content, title: newTitle } : { title: newTitle };\n try {\n await self.helpers.httpRequest({\n method: 'PUT',\n url: `${MINIFLUX_URL}/v1/entries/${e.id}`,\n headers: { 'X-Auth-Token': MINIFLUX_TOKEN, 'Content-Type': 'application/json' },\n body,\n json: true,\n timeout: 15000,\n });\n results.push({ json: { entry_id: e.id, title: newTitle.slice(0, 80), status, source, type, host } });\n } catch (err) {\n results.push({ json: { entry_id: e.id, title: e.title.slice(0, 80), status: 'put_failed', error: String(err).slice(0, 200) } });\n }\n}\n\nresults.push({ json: { stats } });\nreturn results;" }, "typeVersion": 2 }