KG-access
2543b0c23f
- Add Google Scholar Alert collection via Gmail IMAP - Add Gmail App Password based authentication - Add Google Scholar environment variables to NAS compose configuration - Enable Google Scholar Alert source in config.nas.yaml - Disable subject filtering that incorrectly excluded Scholar alert emails - Parse paper titles and links from Google Scholar alert HTML emails - Deduplicate collected Scholar papers by normalized title - Filter Scholar UI/control links such as update alert and unsubscribe entries - Filter bracket-only alert labels such as [automotive radar] - Add Google Scholar specific relevance threshold - Keep global min_relevance at 2 - Set Google Scholar minimum relevance to 3 - Reduce Scholar candidates from 229 collected / 207 merged to 28 accepted - Add Gemini API rate limiting - Enforce minimum 13 second interval between Gemini requests - Apply shared rate limiter to abstract and full-text enrichment - Prevent Gemini free-tier 5 RPM quota errors - Verify retry processing of previously failed AI enrichment jobs - Verify end-to-end NAS workflow - Google Scholar Alert collection successful - Gemini enrichment successful without HTTP 429 errors - Joplin report generation and WebDAV synchronization successful
64 lines
1.6 KiB
Python
64 lines
1.6 KiB
Python
import logging
|
|
|
|
import requests
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class UnpaywallResolver:
|
|
def __init__(self, email: str, timeout=(10, 30)):
|
|
self.email = (email or "").strip()
|
|
self.timeout = timeout
|
|
self.base_url = "https://api.unpaywall.org/v2"
|
|
|
|
def resolve(self, paper):
|
|
if not paper.doi:
|
|
return paper
|
|
|
|
if not self.email:
|
|
log.warning(
|
|
"Unpaywall email missing; OA lookup skipped: %s",
|
|
paper.title,
|
|
)
|
|
return paper
|
|
|
|
doi = paper.doi.strip()
|
|
|
|
try:
|
|
response = requests.get(
|
|
f"{self.base_url}/{doi}",
|
|
params={"email": self.email},
|
|
timeout=self.timeout,
|
|
)
|
|
response.raise_for_status()
|
|
|
|
except requests.exceptions.HTTPError as exc:
|
|
status = (
|
|
exc.response.status_code
|
|
if exc.response is not None
|
|
else None
|
|
)
|
|
|
|
# DOI not present in Unpaywall is not fatal.
|
|
if status == 404:
|
|
paper.oa_status = "not_found"
|
|
return paper
|
|
|
|
raise
|
|
|
|
data = response.json()
|
|
|
|
paper.oa_status = str(
|
|
data.get("oa_status") or ""
|
|
)
|
|
|
|
best = data.get("best_oa_location") or {}
|
|
|
|
pdf_url = best.get("url_for_pdf") or ""
|
|
|
|
# Some OA records do not expose url_for_pdf but do expose a landing URL.
|
|
# We intentionally do NOT treat url as a PDF here.
|
|
paper.pdf_url = str(pdf_url).strip()
|
|
|
|
return paper |