n8n-flows/flows/paywall-bypass.json

{
  "name": "paywall-bypass",
  "nodes": [
    {
      "id": "86fa097b-c230-484d-806d-45b421635c05",
      "name": "Every 5 Minutes",
      "type": "n8n-nodes-base.scheduleTrigger",
      "position": [
        -800,
        0
      ],
      "parameters": {
        "rule": {
          "interval": [
            {
              "field": "minutes",
              "minutesInterval": 5
            }
          ]
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "eb644570-cca3-4464-9b99-02084f199078",
      "name": "Get Miniflux Entries",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        -360,
        0
      ],
      "parameters": {
        "url": "http://192.168.1.40:17002/v1/entries",
        "method": "GET",
        "options": {},
        "sendQuery": true,
        "sendHeaders": true,
        "queryParameters": {
          "parameters": [
            {
              "name": "status",
              "value": "unread"
            },
            {
              "name": "limit",
              "value": "100"
            },
            {
              "name": "order",
              "value": "published_at"
            },
            {
              "name": "direction",
              "value": "desc"
            }
          ]
        },
        "headerParameters": {
          "parameters": [
            {
              "name": "X-Auth-Token",
              "value": "3f439f2db55967ed5acd83398214d2aa76124927cde07115d945b9282597fecb"
            }
          ]
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "f8024262-3163-4cd9-8233-3c91bfddcb01",
      "name": "Process Entries",
      "type": "n8n-nodes-base.code",
      "position": [
        -120,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "const MINIFLUX_TOKEN = '3f439f2db55967ed5acd83398214d2aa76124927cde07115d945b9282597fecb';\nconst MINIFLUX_URL = 'http://192.168.1.40:17002';\n\nconst PAYWALL_NEWS = [\n  'spiegel.de','zeit.de','faz.net','sueddeutsche.de','handelsblatt.com',\n  'nytimes.com','wsj.com','ft.com','washingtonpost.com','thetimes.co.uk',\n  'wired.com','theatlantic.com','economist.com','bloomberg.com',\n  'hbr.org','foreignpolicy.com','newyorker.com','taz.de',\n  'telegraph.co.uk','businessinsider.com'\n];\n\nconst PAYWALL_SCIENCE = [\n  'nature.com','science.org','cell.com','nejm.org','thelancet.com',\n  'link.springer.com','springer.com','wiley.com','onlinelibrary.wiley.com',\n  'sciencedirect.com','pubs.acs.org','rsc.org','pubs.rsc.org',\n  'bmj.com','annualreviews.org','pnas.org',\n  'jamanetwork.com','ahajournals.org','tandfonline.com','academic.oup.com'\n];\n\nconst CONTENT_MIN = 800;\nconst ALREADY_PROCESSED = /[\\u{1F513}\\u{1F4D6}❌]$/u;\n\nfunction extractDoi(url) {\n  let m = url.match(/doi\\.org\\/(10\\.\\d{4,}\\/[^\\s?#]+)/);\n  if (m) return decodeURIComponent(m[1]);\n  m = url.match(/\\/doi\\/(10\\.\\d{4,}\\/[^\\s?#]+)/);\n  if (m) return decodeURIComponent(m[1].replace(/[?#].*$/, ''));\n  return null;\n}\n\nfunction hostname(url) {\n  try { return new URL(url).hostname.replace(/^www\\./, ''); } catch { return ''; }\n}\n\nfunction badge(source, type) {\n  if (source === 'unpaywall') return '<p style=\"font-size:0.8em;color:#555;border-left:3px solid #4CAF50;padding:3px 8px;margin:0 0 1em\">🔓 Open access via Unpaywall</p>';\n  if (type === 'science') return '<p style=\"font-size:0.8em;color:#555;border-left:3px solid #2196F3;padding:3px 8px;margin:0 0 1em\">📖 Full text via crawl4ai (not OA)</p>';\n  return '<p style=\"font-size:0.8em;color:#555;border-left:3px solid #FF9800;padding:3px 8px;margin:0 0 1em\">📖 Paywall bypass via crawl4ai</p>';\n}\n\nconst entries = $('Get Miniflux Entries').first().json.entries || [];\n\nconst results = [];\n\nfor (const e of entries) {\n  if (ALREADY_PROCESSED.test(e.title)) continue;\n\n  const host = hostname(e.url);\n  const isScience = PAYWALL_SCIENCE.some(d => host === d || host.endsWith('.' + d));\n  const isNews = PAYWALL_NEWS.some(d => host === d || host.endsWith('.' + d));\n  const isShort = (e.content || '').replace(/<[^>]+>/g, '').trim().length < CONTENT_MIN;\n\n  if (!isScience && !isNews && !isShort) continue;\n\n  const type = isScience ? 'science' : isNews ? 'news' : 'short';\n  let fetchUrl = e.url;\n  let source = 'crawl4ai';\n\n  if (isScience) {\n    const doi = extractDoi(e.url);\n    if (doi) {\n      try {\n        const upResp = await fetch(\n          `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=fkrebs@nucli.de`\n        );\n        if (upResp.ok) {\n          const up = await upResp.json();\n          if (up.is_oa && up.best_oa_location) {\n            const oaUrl = up.best_oa_location.url || up.best_oa_location.url_for_pdf;\n            if (oaUrl) { fetchUrl = oaUrl; source = 'unpaywall'; }\n          }\n        }\n      } catch (_) {}\n    }\n  }\n\n  let fullContent = '';\n  try {\n    const crResp = await fetch('http://mcp-crawl4ai:11235/md', {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ url: fetchUrl, filter: 'fit' }),\n    });\n    if (crResp.ok) {\n      const cr = await crResp.json();\n      if (cr.success && cr.markdown) {\n        fullContent = '<p>' + cr.markdown.replace(/\\n\\n/g, '</p><p>').replace(/\\n/g, '<br>') + '</p>';\n      }\n    }\n  } catch (_) {}\n\n  if (!fullContent) {\n    const failTitle = e.title.replace(/\\s[\\u{1F513}\\u{1F4D6}❌]$/u, '') + ' ❌';\n    try {\n      await fetch(`${MINIFLUX_URL}/v1/entries/${e.id}`, {\n        method: 'PUT',\n        headers: { 'X-Auth-Token': MINIFLUX_TOKEN, 'Content-Type': 'application/json' },\n        body: JSON.stringify({ title: failTitle }),\n      });\n    } catch (_) {}\n    results.push({ json: { entry_id: e.id, title: failTitle, status: 'no_content', source, type } });\n    continue;\n  }\n\n  const finalContent = badge(source, type) + fullContent;\n  const titleEmoji = source === 'unpaywall' ? '🔓' : '📖';\n  const newTitle = e.title.replace(/\\s[\\u{1F513}\\u{1F4D6}❌]$/u, '') + ' ' + titleEmoji;\n\n  try {\n    await fetch(`${MINIFLUX_URL}/v1/entries/${e.id}`, {\n      method: 'PUT',\n      headers: { 'X-Auth-Token': MINIFLUX_TOKEN, 'Content-Type': 'application/json' },\n      body: JSON.stringify({ content: finalContent, title: newTitle }),\n    });\n    results.push({ json: { entry_id: e.id, title: newTitle, status: 'updated', source, type } });\n  } catch (err) {\n    results.push({ json: { entry_id: e.id, title: e.title, status: 'update_error', error: String(err), source, type } });\n  }\n}\n\nif (results.length === 0) return [{ json: { status: 'no_candidates', checked: entries.length } }];\nreturn results;"
      },
      "typeVersion": 2
    }
  ],
  "connections": {
    "Every 5 Minutes": {
      "main": [
        [
          {
            "node": "Get Miniflux Entries",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get Miniflux Entries": {
      "main": [
        [
          {
            "node": "Process Entries",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "settings": {
    "executionOrder": "v1"
  },
  "id": "86e09d66-edea-4fef-b534-ae1383e954ce"
}