commit c3aad03182000e866daa4e20d8f5efcf475e6e36 Author: yeokwanggoo-stack Date: Sat Aug 15 23:43:07 2026 +0900 First Commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2a18b2b --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +IEEE_API_KEY= +SEMANTIC_SCHOLAR_API_KEY= +OPENAI_API_KEY= +CROSSREF_MAILTO=your_email@example.com +GMAIL_ADDRESS= +GMAIL_APP_PASSWORD= +JOPLIN_TOKEN= diff --git a/.env.nas.example b/.env.nas.example new file mode 100644 index 0000000..009e111 --- /dev/null +++ b/.env.nas.example @@ -0,0 +1,29 @@ +# ========================= +# Paper sources +# ========================= +IEEE_API_KEY= +SEMANTIC_SCHOLAR_API_KEY= +CROSSREF_MAILTO=your_email@example.com +OPENAI_API_KEY= +GMAIL_ADDRESS= +GMAIL_APP_PASSWORD= + +# ========================= +# Joplin WebDAV +# IMPORTANT: Use the same WebDAV sync folder that your existing Joplin clients use. +# Example: https://your-nas-hostname:5006/Joplin +# ========================= +JOPLIN_WEBDAV_URL=https://YOUR_NAS:5006/Joplin +JOPLIN_WEBDAV_USERNAME=joplin-sync +JOPLIN_WEBDAV_PASSWORD=CHANGE_ME + +# Notebook created/used by the NAS automation +JOPLIN_NOTEBOOK=Radar Papers + +# First run: keep false and verify that Joplin CLI can sync existing notes. +# After verification change to true. +JOPLIN_WRITE_ENABLED=false + +# Keep false for a normal valid HTTPS certificate. +# Only consider true if you intentionally use a self-signed certificate. +JOPLIN_IGNORE_TLS_ERRORS=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..05ee58d --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.env +.venv/ +__pycache__/ +*.pyc +data/ +joplin-profile/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3d6bbb3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY src ./src +COPY config.example.yaml ./config.yaml +ENV PYTHONPATH=/app/src +CMD ["python", "-m", "paper_monitor", "--config", "/app/config.yaml"] diff --git a/Dockerfile.nas b/Dockerfile.nas new file mode 100644 index 0000000..9aaa307 --- /dev/null +++ b/Dockerfile.nas @@ -0,0 +1,28 @@ +FROM node:22-bookworm-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/opt/venv/bin:$PATH + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip ca-certificates tini \ + && rm -rf /var/lib/apt/lists/* + +# Joplin Terminal (official npm package) +RUN npm install --loglevel=error -g joplin + +WORKDIR /app +COPY pyproject.toml requirements.txt ./ +COPY src ./src +RUN python3 -m venv /opt/venv \ + && pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir . + +COPY config.nas.yaml /app/config.yaml +COPY nas /app/nas +RUN chmod +x /app/nas/*.sh + +VOLUME ["/app/data", "/joplin-profile"] +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/app/nas/run_once.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..920c075 --- /dev/null +++ b/README.md @@ -0,0 +1,136 @@ +# Paper Monitor → Joplin + +자동차 레이더 관련 신규 논문을 IEEE Xplore, Semantic Scholar, Crossref, Google Scholar Alert에서 수집하고 DOI/제목 기준으로 중복 제거한 뒤 Markdown 표와 Joplin note로 만드는 프로젝트입니다. + +## 빠른 시작 + +```bash +cd paper-monitor +cp config.example.yaml config.yaml +cp .env.example .env +python -m venv .venv +# Linux/macOS: source .venv/bin/activate +# Windows: .venv\Scripts\activate +pip install -e . +``` + +환경변수에 필요한 키를 설정합니다. + +```text +IEEE_API_KEY=... +SEMANTIC_SCHOLAR_API_KEY=... +CROSSREF_MAILTO=you@example.com +JOPLIN_TOKEN=... +``` + +실행: + +```bash +python -m paper_monitor --config config.yaml --dry-run +python -m paper_monitor --config config.yaml +``` + +`--dry-run`은 Joplin 쓰기만 생략합니다. SQLite와 Markdown 생성은 수행합니다. + +## 동작 + +1. `config.yaml`의 각 검색어를 소스별로 조회 +2. 최근 `lookback_days` 필터 +3. DOI 우선, DOI 없으면 normalized-title hash로 중복 제거 +4. category와 relevance 1~5 계산 +5. 선택적으로 OpenAI API로 한국어 요약/relevance/category 보강 +6. `data/papers.db`에 이미 본 논문 저장 +7. 처음 발견한 논문만 `data/outbox/*.md`에 출력 +8. Joplin Data API가 사용 가능하면 `Research/Radar Papers` note 자동 생성/갱신 + +## Google Scholar + +Scholar 검색 결과 HTML을 직접 scraping하지 않습니다. Google Scholar Alert를 Gmail로 받은 뒤 IMAP으로 읽습니다. `google_scholar_alert.enabled: true`로 켜고 아래 변수를 설정합니다. + +```text +GMAIL_ADDRESS=... +GMAIL_APP_PASSWORD=... +``` + +Google은 일반 계정 비밀번호를 IMAP 클라이언트에 직접 공유하는 방식을 권장하지 않습니다. 개인 계정에서 사용할 수 있는 경우 2단계 인증 + App Password를 쓰거나, 장기 운영에서는 Gmail OAuth/Gmail API로 collector를 교체하는 편이 좋습니다. + +## Joplin + +기본 출력은 Joplin Data API입니다. Joplin Desktop에서 Web Clipper service를 켜고 token을 발급한 뒤: + +```yaml +joplin: + enabled: true + base_url: http://127.0.0.1:41184 + token_env: JOPLIN_TOKEN +``` + +Joplin API가 실패해도 Markdown은 항상 `data/outbox`에 남습니다. + +### NAS에서 주의 + +NAS Docker 내부의 `127.0.0.1:41184`는 Windows PC가 아니라 NAS 컨테이너 자신입니다. 따라서 가장 간단한 1차 구성은 **Windows Task Scheduler + Joplin Desktop/Web Clipper + 이 프로그램**입니다. 논문 수집만 NAS에서 돌리고 Markdown을 NAS에 보관하는 것도 가능합니다. + +Joplin 공식 Terminal 앱은 WebDAV sync와 cron 실행을 지원하므로, 향후 완전 NAS-only 구성으로 바꿀 때 사용할 수 있습니다. + +## AI 요약 + +기본은 off입니다. + +```yaml +ai: + enabled: true + api_key_env: OPENAI_API_KEY + model: gpt-5-mini + max_papers_per_run: 30 +``` + +AI 호출이 실패해도 keyword 기반 결과로 계속 진행됩니다. + +## Docker / Synology Container Manager + +```bash +cp config.example.yaml config.yaml +cp .env.example .env +docker compose build +docker compose run --rm paper-monitor +``` + +매일 06:00 cron 예: + +```cron +0 6 * * * cd /volume1/docker/paper-monitor && /usr/bin/docker compose run --rm paper-monitor >> ./data/cron.log 2>&1 +``` + +Synology Task Scheduler에서도 같은 명령을 User-defined script로 등록할 수 있습니다. 실제 docker 경로는 NAS에서 `which docker`로 확인하세요. + +## 결과 형식 + +```markdown +| Relevance | Category | Paper | Authors | Venue | Date | Citations | Source | +|---|---|---|---|---|---|---:|---| +| ★★★★★ | Imaging Radar, MIMO | [Paper title](...) | ... | IEEE TAP | 2026-08-11 | 2 | IEEE Xplore | +``` + +표 아래에 논문별 DOI, 링크, abstract excerpt 또는 AI summary가 추가됩니다. + +## 권장 초기 설정 + +처음 1~2주는 AI와 Scholar Alert를 끄고 IEEE + Semantic Scholar + Crossref만 사용해 noise를 확인하는 것을 권장합니다. + +```yaml +app: + lookback_days: 14 + min_relevance: 2 +ai: + enabled: false +sources: + google_scholar_alert: + enabled: false +``` + +그 후 `queries`, `categories`, `relevance_terms`를 본인 관심 분야에 맞게 조정하면 됩니다. + +## 보안 + +API key, Gmail App Password, Joplin token은 Git에 커밋하지 마세요. 환경변수/Docker secret 등을 사용하세요. diff --git a/README_NAS.md b/README_NAS.md new file mode 100644 index 0000000..e08e29c --- /dev/null +++ b/README_NAS.md @@ -0,0 +1,383 @@ +# Synology NAS-only Paper Monitor → Joplin + +이 구성은 Windows PC가 꺼져 있어도 Synology NAS만으로 다음 작업을 수행합니다. + +```text +Paper APIs + ↓ +Python paper-monitor + ↓ +SQLite deduplication + ↓ +Daily Markdown report + ↓ +Joplin Terminal import + ↓ +Joplin WebDAV sync + ↓ +Synology WebDAV sync folder + ↓ +PC / phone / tablet Joplin +``` + +## 중요한 원칙 + +NAS의 Joplin WebDAV 동기화 폴더에 Markdown 파일을 직접 복사하지 않습니다. +반드시 Joplin Terminal을 통해 note를 만들고 `joplin sync`를 실행합니다. + +`joplin-profile/`은 NAS의 Joplin CLI 전용 로컬 profile입니다. PC Joplin profile과 공유하지 마세요. + +--- + +## 1. NAS에 프로젝트 복사 + +예시: + +```text +/volume1/docker/paper-monitor/ +``` + +이 폴더에 이 ZIP의 전체 파일을 복사합니다. + +최소 파일: + +```text +paper-monitor/ +├── Dockerfile.nas +├── docker-compose.nas.yml +├── config.nas.yaml +├── .env +├── src/ +├── nas/ +├── data/ +└── joplin-profile/ +``` + +`data`와 `joplin-profile` 폴더는 없으면 Docker 실행 시 생성할 수 있지만, 미리 만드는 것을 권장합니다. + +```bash +mkdir -p /volume1/docker/paper-monitor/data +mkdir -p /volume1/docker/paper-monitor/joplin-profile +``` + +--- + +## 2. .env 생성 + +`.env.nas.example`을 복사하여 `.env`로 만듭니다. + +```bash +cp .env.nas.example .env +``` + +예: + +```env +IEEE_API_KEY= +SEMANTIC_SCHOLAR_API_KEY= +CROSSREF_MAILTO=your_email@example.com + +JOPLIN_WEBDAV_URL=https://your-nas-hostname:5006/Joplin +JOPLIN_WEBDAV_USERNAME=joplin-sync +JOPLIN_WEBDAV_PASSWORD=YOUR_PASSWORD +JOPLIN_NOTEBOOK=Radar Papers +JOPLIN_WRITE_ENABLED=false +JOPLIN_IGNORE_TLS_ERRORS=false +``` + +### JOPLIN_WEBDAV_URL + +현재 PC/스마트폰 Joplin에서 사용 중인 **동일한 Joplin WebDAV sync folder**를 지정해야 합니다. + +예: + +```text +https://your-nas-hostname:5006/Joplin +``` + +NAS Docker 컨테이너에서 해당 주소에 접근할 수 있어야 합니다. + +### 보안 + +`.env`에는 WebDAV 비밀번호와 API key가 있으므로 Git에 올리지 마세요. + +--- + +## 3. 처음에는 쓰기 기능을 끈 상태로 실행 + +처음에는 반드시: + +```env +JOPLIN_WRITE_ENABLED=false +``` + +로 두는 것을 권장합니다. + +이 상태에서는 **기존 Joplin WebDAV 데이터를 Joplin CLI profile로 동기화만 하고 새 note는 만들지 않습니다.** + +가능하면 첫 테스트 전에 NAS의 Joplin WebDAV 폴더를 백업하세요. + +--- + +## 4. Docker image 빌드 + +NAS SSH에서: + +```bash +cd /volume1/docker/paper-monitor +sudo docker compose -f docker-compose.nas.yml build +``` + +Joplin Terminal npm package와 Python paper-monitor가 한 image에 설치됩니다. + +--- + +## 5. Joplin WebDAV sync-only 테스트 + +```bash +sudo docker compose -f docker-compose.nas.yml run --rm paper-monitor-nas /app/nas/test_sync.sh +``` + +정상적이면 `joplin sync`가 완료되고 Joplin status가 출력됩니다. + +이때 다음 폴더에 NAS용 Joplin local profile이 저장됩니다. + +```text +/volume1/docker/paper-monitor/joplin-profile/ +``` + +이 profile은 이후 실행에서도 유지됩니다. + +--- + +## 6. 논문 자동 작성 활성화 + +sync-only 테스트가 정상이라면 `.env`를 수정합니다. + +```env +JOPLIN_WRITE_ENABLED=true +``` + +현재 IEEE key가 아직 Waiting이라면 `config.nas.yaml`에서: + +```yaml +sources: + ieee: + enabled: false +``` + +Semantic Scholar key도 아직 승인되지 않았다면: + +```yaml + semantic_scholar: + enabled: false +``` + +Crossref만으로도 전체 동작을 시험할 수 있습니다. + +--- + +## 7. 1회 전체 테스트 + +```bash +sudo docker compose -f docker-compose.nas.yml run --rm paper-monitor-nas +``` + +동작 순서: + +```text +1. Joplin CLI → WebDAV sync (remote 변경 먼저 가져오기) +2. 논문 API 검색 +3. SQLite 중복 제거 +4. 오늘 발견한 논문으로 일일 누적 Markdown 생성 +5. Joplin의 동일 날짜 report가 있으면 교체 +6. Markdown을 Joplin notebook으로 import +7. Joplin CLI → WebDAV sync +``` + +정상 로그 예: + +```text +Synchronising Joplin from WebDAV before writing... +Running paper monitor... +Crossref 'automotive radar' -> 10 +... +Importing report into Joplin notebook: Radar Papers +Synchronising Joplin changes to WebDAV... +Completed: Radar Literature - 2026-08-12 (... papers) +``` + +Joplin에서 다음 note가 보이면 성공입니다. + +```text +Radar Papers +└── Radar Literature - 2026-08-12 +``` + +--- + +## 8. 같은 날 여러 번 실행해도 괜찮은 이유 + +`papers.db`에 논문별 최초 발견 날짜를 저장합니다. + +예: + +```text +06:00 실행: 논문 A, B 발견 +→ 오늘 report = A, B + +12:00 재실행: 논문 C 추가 발견 +→ 오늘 report = A, B, C +``` + +따라서 같은 날짜의 Joplin report를 교체해도 앞서 발견한 논문이 사라지지 않습니다. + +--- + +## 9. Synology Task Scheduler에서 매일 자동 실행 + +DSM: + +```text +Control Panel +→ Task Scheduler +→ Create +→ Scheduled Task +→ User-defined script +``` + +예를 들어 매일 오전 06:00으로 설정합니다. + +User-defined script: + +```bash +/volume1/docker/paper-monitor/run_scheduled.sh +``` + +NAS에 따라 Docker 경로가 다를 수 있습니다. + +SSH에서 확인: + +```bash +which docker +``` + +출력된 경로로 `/usr/local/bin/docker`를 교체하세요. + +--- + +## 10. 자동 실행 로그 확인 + +```bash +ls -1t /volume1/docker/paper-monitor/data/logs/scheduler-*.log | head -n 1 +tail -n 200 "$(ls -1t /volume1/docker/paper-monitor/data/logs/scheduler-*.log | head -n 1)" +``` + +논문 수집 결과: + +```text +/volume1/docker/paper-monitor/data/papers.db +/volume1/docker/paper-monitor/data/outbox/ +/volume1/docker/paper-monitor/data/last_result.json +``` + +--- + +## 11. API 활성화 + +### IEEE + +IEEE developer application이 Active가 된 뒤: + +`.env`: + +```env +IEEE_API_KEY=YOUR_KEY +``` + +`config.nas.yaml`: + +```yaml +ieee: + enabled: true +``` + +401/403이 발생하면 해당 실행에서 IEEE query는 한 번 실패한 뒤 중단하도록 수정되어 있습니다. + +### Semantic Scholar + +API key 승인 후: + +```env +SEMANTIC_SCHOLAR_API_KEY=YOUR_KEY +``` + +그리고: + +```yaml +semantic_scholar: + enabled: true +``` + +HTTP 429가 발생하면 exponential backoff로 재시도하고, 계속 제한되면 해당 실행의 Semantic Scholar query를 중단합니다. + +--- + +## 12. HTTPS 인증서 문제 + +정상적인 공인 인증서를 사용한다면: + +```env +JOPLIN_IGNORE_TLS_ERRORS=false +``` + +를 유지하세요. + +자체 서명 인증서 때문에 sync가 실패할 때만 임시 진단 목적으로: + +```env +JOPLIN_IGNORE_TLS_ERRORS=true +``` + +를 사용할 수 있습니다. 가능하면 인증서를 정상 구성하는 것이 우선입니다. + +--- + +## 13. Joplin E2EE를 사용하는 경우 + +기존 Joplin 동기화에 End-to-End Encryption을 활성화해 둔 경우, 새 NAS Joplin CLI client에서도 master key/password 설정이 필요할 수 있습니다. + +그 경우 자동 쓰기를 켜기 전에 Joplin CLI의 E2EE 상태와 복호화를 먼저 설정해야 합니다. + +```bash +sudo docker compose -f docker-compose.nas.yml run --rm paper-monitor-nas \ + joplin --profile /joplin-profile e2ee status +``` + +E2EE를 사용하지 않는 경우에는 이 단계가 필요 없습니다. + +--- + +## 14. 운영 시 권장 구성 + +```yaml +app: + lookback_days: 14 + min_relevance: 2 + +sources: + ieee: + enabled: true # 승인 후 + semantic_scholar: + enabled: true # 승인 후 + crossref: + enabled: true + +ai: + enabled: false # 처음에는 비용 없는 상태로 운영 + +joplin: + enabled: false # NAS에서는 Joplin CLI wrapper가 담당 +``` + +Joplin Data API용 `joplin.py`는 Windows Desktop 방식 호환을 위해 프로젝트에 유지하지만 NAS 경로에서는 사용하지 않습니다. diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..de6c46d --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,82 @@ +app: + timezone: Asia/Seoul + lookback_days: 14 + max_papers_per_source_per_query: 50 + min_relevance: 2 + output_dir: ./data/outbox + database_path: ./data/papers.db + log_level: INFO + +search: + queries: + - automotive radar + - automotive imaging radar + - 4D radar + - FMCW radar MIMO + - radar DOA estimation + - wideband DOA radar + - automotive radar antenna + - radar target simulation + - radar interference + - radar phase noise + categories: + Antenna: [antenna, array, gap waveguide, gapwaveguide, radome, beamforming, radiation pattern, sidelobe] + Waveform: [waveform, FMCW, chirp, stepped frequency, frequency step, PMCW, phase coded] + DOA: [DOA, direction of arrival, angle estimation, MUSIC, ESPRIT, angle FFT, beamspace] + MIMO: [MIMO, TDM, BPM, DDMA, virtual array, multiplexing] + Target Modeling: [target model, target simulation, extended target, scatterer, scattering center, Swerling, RCS] + Interference: [interference, mutual interference, interference mitigation, coexistence] + RF Impairment: [phase noise, nonlinearity, IQ imbalance, leakage, oscillator] + Imaging Radar: [imaging radar, 4D radar, high resolution radar, point cloud] + relevance_terms: + automotive radar: 3 + imaging radar: 3 + 4D radar: 3 + FMCW: 2 + MIMO: 2 + DOA: 2 + direction of arrival: 2 + antenna: 1 + target simulation: 2 + phase noise: 2 + interference: 1 + +sources: + ieee: + enabled: true + api_key_env: IEEE_API_KEY + base_url: https://ieeexploreapi.ieee.org/api/v1/search/articles + semantic_scholar: + enabled: true + api_key_env: SEMANTIC_SCHOLAR_API_KEY + base_url: https://api.semanticscholar.org/graph/v1/paper/search + crossref: + enabled: true + mailto_env: CROSSREF_MAILTO + base_url: https://api.crossref.org/works + google_scholar_alert: + enabled: false + gmail_address_env: GMAIL_ADDRESS + gmail_app_password_env: GMAIL_APP_PASSWORD + imap_host: imap.gmail.com + mailbox: INBOX + sender_contains: scholaralerts-noreply + subject_contains: Google Scholar + +ai: + enabled: false + provider: openai + api_key_env: OPENAI_API_KEY + model: gpt-5-mini + max_papers_per_run: 30 + language: ko + +joplin: + enabled: true + base_url: http://127.0.0.1:41184 + token_env: JOPLIN_TOKEN + notebook_path: [Research, Radar Papers] + note_mode: daily + note_title_prefix: Radar Literature + update_existing_note: true + tags: [literature-monitor, automotive-radar] diff --git a/config.nas.yaml b/config.nas.yaml new file mode 100644 index 0000000..ce1ab3e --- /dev/null +++ b/config.nas.yaml @@ -0,0 +1,90 @@ +app: + timezone: Asia/Seoul + lookback_days: 14 + max_papers_per_source_per_query: 20 + min_relevance: 2 + output_dir: ./data/outbox + database_path: ./data/papers.db + log_level: INFO + +search: + queries: + - automotive radar + - automotive imaging radar + - 4D radar + - FMCW radar MIMO + - radar DOA estimation + - wideband DOA radar + - automotive radar antenna + - radar target simulation + - radar interference + - super resolution DOA + + categories: + Antenna: [antenna, array, radome, beamforming, radiation pattern, sidelobe] + Waveform: [waveform, FMCW, chirp, stepped frequency, frequency step, PMCW, phase coded] + DOA: [DOA, direction of arrival, angle estimation, MUSIC, ESPRIT, angle FFT, beamspace, super resolution] + MIMO: [MIMO, TDM, BPM, DDMA, virtual array, multiplexing] + Target Modeling: [target model, target simulation, extended target, scatterer, scattering center, Swerling, RCS] + Interference: [interference, mutual interference, interference mitigation, coexistence] + RF Impairment: [phase noise, nonlinearity, IQ imbalance, leakage, oscillator] + Imaging Radar: [imaging radar, 4D radar, high resolution radar, point cloud] + + relevance_terms: + automotive radar: 3 + imaging radar: 3 + 4D radar: 3 + FMCW: 2 + MIMO: 2 + DOA: 2 + direction of arrival: 2 + antenna: 1 + target simulation: 2 + phase noise: 2 + interference: 1 + +sources: + ieee: + enabled: false # Set true after the IEEE developer application becomes Active. + api_key_env: IEEE_API_KEY + base_url: https://ieeexploreapi.ieee.org/api/v1/search/articles + + semantic_scholar: + enabled: false # Set true after your API key is approved/available. + api_key_env: SEMANTIC_SCHOLAR_API_KEY + base_url: https://api.semanticscholar.org/graph/v1/paper/search + + crossref: + enabled: true + mailto_env: CROSSREF_MAILTO + base_url: https://api.crossref.org/works + + google_scholar_alert: + enabled: false + gmail_address_env: GMAIL_ADDRESS + gmail_app_password_env: GMAIL_APP_PASSWORD + imap_host: imap.gmail.com + mailbox: INBOX + sender_contains: scholaralerts-noreply + subject_contains: Google Scholar + +ai: + enabled: true + provider: gemini + api_key_env: GEMINI_API_KEY + model: gemini-3.6-flash + max_papers_per_run: 20 + +# NAS mode writes to Joplin through Joplin Terminal, not the desktop Data API. +joplin: + enabled: false + note_mode: daily + note_title_prefix: Radar Literature + +# OA status and PDF URL retrieval through Unpaywall API. +oa: + enabled: true + provider: unpaywall + email_env: UNPAYWALL_EMAIL + + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..a617bc4 --- /dev/null +++ b/config.yaml @@ -0,0 +1,82 @@ +app: + timezone: Asia/Seoul + lookback_days: 14 + max_papers_per_source_per_query: 50 + min_relevance: 2 + output_dir: ./data/outbox + database_path: ./data/papers.db + log_level: INFO + +search: + queries: + - automotive radar + - automotive imaging radar + - 4D radar + - FMCW radar MIMO + - radar DOA estimation + - wideband DOA radar + - automotive radar antenna + - radar target simulation + - radar interference + - radar phase noise + categories: + Antenna: [antenna, array, gap waveguide, gapwaveguide, radome, beamforming, radiation pattern, sidelobe] + Waveform: [waveform, FMCW, chirp, stepped frequency, frequency step, PMCW, phase coded] + DOA: [DOA, direction of arrival, angle estimation, MUSIC, ESPRIT, angle FFT, beamspace] + MIMO: [MIMO, TDM, BPM, DDMA, virtual array, multiplexing] + Target Modeling: [target model, target simulation, extended target, scatterer, scattering center, Swerling, RCS] + Interference: [interference, mutual interference, interference mitigation, coexistence] + RF Impairment: [phase noise, nonlinearity, IQ imbalance, leakage, oscillator] + Imaging Radar: [imaging radar, 4D radar, high resolution radar, point cloud] + relevance_terms: + automotive radar: 3 + imaging radar: 3 + 4D radar: 3 + FMCW: 2 + MIMO: 2 + DOA: 2 + direction of arrival: 2 + antenna: 1 + target simulation: 2 + phase noise: 2 + interference: 1 + +sources: + ieee: + enabled: false + api_key_env: IEEE_API_KEY + base_url: https://ieeexploreapi.ieee.org/api/v1/search/articles + semantic_scholar: + enabled: true + api_key_env: SEMANTIC_SCHOLAR_API_KEY + base_url: https://api.semanticscholar.org/graph/v1/paper/search + crossref: + enabled: false + mailto_env: CROSSREF_MAILTO + base_url: https://api.crossref.org/works + google_scholar_alert: + enabled: false + gmail_address_env: GMAIL_ADDRESS + gmail_app_password_env: GMAIL_APP_PASSWORD + imap_host: imap.gmail.com + mailbox: INBOX + sender_contains: scholaralerts-noreply + subject_contains: Google Scholar + +ai: + enabled: false + provider: openai + api_key_env: OPENAI_API_KEY + model: gpt-5-mini + max_papers_per_run: 30 + language: ko + +joplin: + enabled: true + base_url: http://127.0.0.1:41184 + token_env: JOPLIN_TOKEN + notebook_path: [Research, Radar Papers] + note_mode: daily + note_title_prefix: Radar Literature + update_existing_note: true + tags: [literature-monitor, automotive-radar] diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml new file mode 100644 index 0000000..166df34 --- /dev/null +++ b/docker-compose.nas.yml @@ -0,0 +1,40 @@ +services: + paper-monitor-nas: + build: + context: . + dockerfile: Dockerfile.nas + image: paper-monitor-nas:latest + container_name: paper-monitor-nas-run + restart: "no" + environment: + TZ: Asia/Seoul + + CIRCUIT_BREAKER_STATE: /state/provider_circuit_breaker_state.json + + # Paper APIs + IEEE_API_KEY: ${IEEE_API_KEY:-} + SEMANTIC_SCHOLAR_API_KEY: ${SEMANTIC_SCHOLAR_API_KEY:-} + CROSSREF_MAILTO: ${CROSSREF_MAILTO:-} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + GEMINI_API_KEY: ${GEMINI_API_KEY} + GROQ_API_KEY: ${GROQ_API_KEY} + GMAIL_ADDRESS: ${GMAIL_ADDRESS:-} + GMAIL_APP_PASSWORD: ${GMAIL_APP_PASSWORD:-} + + # Existing Joplin WebDAV sync target + JOPLIN_WEBDAV_URL: ${JOPLIN_WEBDAV_URL} + JOPLIN_WEBDAV_USERNAME: ${JOPLIN_WEBDAV_USERNAME} + JOPLIN_WEBDAV_PASSWORD: ${JOPLIN_WEBDAV_PASSWORD} + JOPLIN_NOTEBOOK: ${JOPLIN_NOTEBOOK:-Radar Papers} + JOPLIN_WRITE_ENABLED: ${JOPLIN_WRITE_ENABLED:-false} + JOPLIN_IGNORE_TLS_ERRORS: ${JOPLIN_IGNORE_TLS_ERRORS:-false} + + # Unpaywall E-Mail address for OA status and PDF URL retrieval. + # This is required for the Unpaywall API. + UNPAYWALL_EMAIL: ${UNPAYWALL_EMAIL} + + volumes: + - ./config.nas.yaml:/app/config.yaml:ro + - ./data:/app/data + - ./joplin-profile:/joplin-profile + - ./state:/state diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..14cef8f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + paper-monitor: + build: . + container_name: paper-monitor + restart: "no" + environment: + - IEEE_API_KEY=${IEEE_API_KEY} + - SEMANTIC_SCHOLAR_API_KEY=${SEMANTIC_SCHOLAR_API_KEY} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - CROSSREF_MAILTO=${CROSSREF_MAILTO} + - GMAIL_ADDRESS=${GMAIL_ADDRESS} + - GMAIL_APP_PASSWORD=${GMAIL_APP_PASSWORD} + - JOPLIN_TOKEN=${JOPLIN_TOKEN} + volumes: + - ./config.yaml:/app/config.yaml:ro + - ./data:/app/data + command: ["python", "-m", "paper_monitor", "--config", "/app/config.yaml"] diff --git a/nas/run_once.sh b/nas/run_once.sh new file mode 100644 index 0000000..9b9406e --- /dev/null +++ b/nas/run_once.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +PROFILE="/joplin-profile" +CONFIG="/app/config.yaml" +RESULT_JSON="/app/data/last_result.json" + +log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; } +fail() { log "ERROR: $*"; exit 1; } + +: "${JOPLIN_WEBDAV_URL:?JOPLIN_WEBDAV_URL is required}" +: "${JOPLIN_WEBDAV_USERNAME:?JOPLIN_WEBDAV_USERNAME is required}" +: "${JOPLIN_WEBDAV_PASSWORD:?JOPLIN_WEBDAV_PASSWORD is required}" + +JOPLIN_NOTEBOOK="${JOPLIN_NOTEBOOK:-Radar Papers}" +JOPLIN_WRITE_ENABLED="${JOPLIN_WRITE_ENABLED:-false}" +JOPLIN_IGNORE_TLS_ERRORS="${JOPLIN_IGNORE_TLS_ERRORS:-false}" + +mkdir -p "$PROFILE" /app/data/outbox + +log "Configuring Joplin CLI WebDAV profile..." +joplin --profile "$PROFILE" config sync.target 6 >/dev/null +joplin --profile "$PROFILE" config sync.6.path "$JOPLIN_WEBDAV_URL" >/dev/null +joplin --profile "$PROFILE" config sync.6.username "$JOPLIN_WEBDAV_USERNAME" >/dev/null +joplin --profile "$PROFILE" config sync.6.password "$JOPLIN_WEBDAV_PASSWORD" >/dev/null +joplin --profile "$PROFILE" config sync.wipeOutFailSafe true >/dev/null + +if [[ "${JOPLIN_IGNORE_TLS_ERRORS,,}" == "true" ]]; then + joplin --profile "$PROFILE" config net.ignoreTlsErrors true >/dev/null +else + joplin --profile "$PROFILE" config net.ignoreTlsErrors false >/dev/null +fi + +# Pull the current remote state first. This is important when PC/mobile also use the same WebDAV target. +log "Synchronising Joplin from WebDAV before writing..." +joplin --profile "$PROFILE" sync + +if [[ "${JOPLIN_WRITE_ENABLED,,}" != "true" ]]; then + log "JOPLIN_WRITE_ENABLED=false: sync-only safety mode. No paper note will be written." + log "If this first sync succeeded, set JOPLIN_WRITE_ENABLED=true and run again." + exit 0 +fi + +log "Running paper monitor..." +python -m paper_monitor --config "$CONFIG" --dry-run +[[ -f "$RESULT_JSON" ]] || fail "Result JSON not produced: $RESULT_JSON" + +readarray -t META < <(python - "$RESULT_JSON" <<'PY' +import json, sys +p=sys.argv[1] +d=json.load(open(p,encoding='utf-8')) +print(d.get('report_count',0)) +print(d.get('report_title','')) +print(d.get('markdown','')) +PY +) + +REPORT_COUNT="${META[0]:-0}" +REPORT_TITLE="${META[1]:-}" +MARKDOWN="${META[2]:-}" + +if [[ "$REPORT_COUNT" == "0" ]]; then + log "No papers in today's report. Nothing to write to Joplin." + exit 0 +fi + +[[ -n "$REPORT_TITLE" ]] || fail "Empty report title" +[[ -f "$MARKDOWN" ]] || fail "Markdown report not found: $MARKDOWN" + +log "Ensuring notebook exists: $JOPLIN_NOTEBOOK" +if ! joplin --profile "$PROFILE" use "$JOPLIN_NOTEBOOK" >/dev/null 2>&1; then + joplin --profile "$PROFILE" mkbook "$JOPLIN_NOTEBOOK" +fi + +# Replace only the daily report with the exact same title. Since the report is cumulative +# for the local day, rerunning is idempotent and will not lose earlier papers from the same day. +log "Replacing existing daily note if present: $REPORT_TITLE" +joplin --profile "$PROFILE" rmnote "$REPORT_TITLE" -f >/dev/null 2>&1 || true + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT +TMP_MD="$TMP_DIR/$REPORT_TITLE.md" +cp "$MARKDOWN" "$TMP_MD" + +log "Importing report into Joplin notebook: $JOPLIN_NOTEBOOK" +joplin --profile "$PROFILE" import "$TMP_MD" "$JOPLIN_NOTEBOOK" --format md -f + +log "Synchronising Joplin changes to WebDAV..." +joplin --profile "$PROFILE" sync + +log "Completed: $REPORT_TITLE ($REPORT_COUNT papers)" diff --git a/nas/test_sync.sh b/nas/test_sync.sh new file mode 100644 index 0000000..8477931 --- /dev/null +++ b/nas/test_sync.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +PROFILE="/joplin-profile" +: "${JOPLIN_WEBDAV_URL:?JOPLIN_WEBDAV_URL is required}" +: "${JOPLIN_WEBDAV_USERNAME:?JOPLIN_WEBDAV_USERNAME is required}" +: "${JOPLIN_WEBDAV_PASSWORD:?JOPLIN_WEBDAV_PASSWORD is required}" +mkdir -p "$PROFILE" +joplin --profile "$PROFILE" config sync.target 6 >/dev/null +joplin --profile "$PROFILE" config sync.6.path "$JOPLIN_WEBDAV_URL" >/dev/null +joplin --profile "$PROFILE" config sync.6.username "$JOPLIN_WEBDAV_USERNAME" >/dev/null +joplin --profile "$PROFILE" config sync.6.password "$JOPLIN_WEBDAV_PASSWORD" >/dev/null +joplin --profile "$PROFILE" config sync.wipeOutFailSafe true >/dev/null +joplin --profile "$PROFILE" sync +joplin --profile "$PROFILE" status diff --git a/paper-monitor-todo.md b/paper-monitor-todo.md new file mode 100644 index 0000000..a2ff617 --- /dev/null +++ b/paper-monitor-todo.md @@ -0,0 +1,556 @@ +# Paper Monitor 프로젝트 - 남은 작업 및 검토 TODO + +최종 업데이트: 2026-08-15 + +이 문서는 지금까지 구축한 Synology NAS 기반 논문 자동 모니터링 시스템에서 **아직 미정이거나 수행하지 못한 항목**, 그리고 **운영 전에 변경 또는 추가 검토가 필요한 항목**을 정리한 TODO 목록이다. + +--- + +## 1. 즉시 처리 권장 + +### [ ] Semantic Scholar 차단 해제 후 재활성화 +- 현재 상태: 접근 제한/차단 상태이므로 `enabled: false` 권장 +- 차단이 풀린 뒤 다시 활성화 +- 재활성화 후 확인 + - 2.5초 이상 요청 간격이 실제 적용되는지 + - circuit breaker가 429 지속 시 정상 동작하는지 + - 한 run에서 과도한 재시도가 발생하지 않는지 +- 정상화 후에도 429가 잦으면 검토 + - 요청 간격 3~5초로 증가 + - 검색 query 수 축소 + - `max_retries` 추가 축소 + +### [ ] IEEE 접근 차단 해제 여부 확인 +- 현재 상태 + - 집 인터넷: `developer.ieee.org` -> HTTP 403 + - 동일 PC + 휴대폰 테더링: HTTP 200 OK + - 현재 공인 IP 또는 해당 회선이 IEEE/Mashery/WAF에서 제한된 가능성이 매우 높음 +- 현재는 `IEEE enabled: false` 유지 +- 차단 해제 후에만 다시 활성화 +- 재활성화 시 먼저 1개 query만 수동 테스트 +- 401 / 403 / 429 / timeout / connection error 발생 시 해당 run의 IEEE 전체 중단 유지 +- 동일 run에서 IEEE 자동 retry는 하지 않음 +- 요청 간 최소 간격 5초 유지 + +### [ ] IEEE API 승인 상태 확인 +- 네트워크 차단 문제와 API 승인 문제는 별개 +- IEEE API key가 최종 승인되었는지 확인 후 활성화 +- 웹사이트 접속 정상화와 API 승인 상태를 각각 확인 + +### [ ] 노출된 IEEE API key 교체 여부 확인 +- 과거 오류 로그에 API key가 query string 형태로 노출된 적이 있음 +- 아직 교체하지 않았다면 새 API key 발급/회전 권장 +- 현재 수정한 IEEE collector는 향후 오류 로그에서 key가 노출되지 않도록 유지 + +--- + +## 2. 스케줄러 / 운영 설정 + +현재 미완료 항목 없음. 완료된 내용은 문서 하단 `완료 항목` 섹션 참조. + +--- + +## 3. 검색 범위 및 수집 정책 + +### [ ] `lookback_days: 14` 유지 여부 장기 검토 +현재: +```yaml +lookback_days: 14 +``` + +주 1회 실행에는 안전한 값이며 현재는 유지 권장. + +향후 확인: +- 매주 중복 수집량이 지나치게 많으면 10일 정도로 감소 검토 +- publication date가 늦게 반영되는 source가 많으면 14일 유지 + +### [ ] Query 목록 재정비 +현재 자동차 레이더 관련 다수 query 사용. + +검토 필요: +- 중복도가 지나치게 높은 query 제거 +- 실제 유용 논문을 거의 만들지 않는 query 제거 +- 주요 관심영역별 query 분리 + +예: +- Automotive radar +- Imaging radar / 4D radar +- FMCW / Waveform +- MIMO / DDMA +- DOA / Wideband DOA / Super-resolution +- Antenna +- Target modeling / Scattering center +- Interference / RF impairment + +### [ ] Source별 query 목록 분리 검토 +IEEE, Semantic Scholar, Crossref에서 동일한 query 세트를 사용하는 대신 source 특성에 맞게 별도 query 목록을 둘지 검토. + +예: +```yaml +sources: + crossref: + queries: [...] + semantic_scholar: + queries: [...] + ieee: + queries: [...] +``` + +장점: +- API 호출량 감소 +- 불필요한 중복 감소 +- IEEE와 Semantic Scholar 차단 가능성 감소 + +--- + +## 4. Gemini AI 운영 설정 + +### [ ] Gemini 무료 quota 실제 사용량 모니터링 +확인 항목: +- full-text 요청 수 +- abstract-only 요청 수 +- 한 run에서 token 사용량 +- quota 초과/429 발생 여부 + +필요 시: +- full-text 분석 대상 relevance threshold 추가 +- 예: rule-based 또는 abstract AI relevance가 높은 논문만 full-text 분석 + +### [ ] Full-text AI 2단계 분석 구조 검토 +현재는 OA PDF가 있으면 바로 full-text 분석. + +향후 비용 절감 구조 검토: +1. 모든 신규 논문 -> title + abstract 분석 +2. relevance >= 특정 값인 논문만 OA PDF full-text 분석 + +예: +```text +abstract AI relevance >= 4/5 + -> full-text AI +else + -> abstract 결과만 유지 +``` + +장점: +- Gemini 호출량/토큰 절감 +- PDF 다운로드/파싱 횟수 감소 + +### [ ] Gemini 모델명 config 기반 유지 +현재 `gemini-3.6-flash` 사용. + +향후 모델 변경 가능성을 고려해 hard-code 최소화. +`config.nas.yaml` 값만 바꾸면 동작하도록 유지. + +### [ ] Gemini AFC 경고 정리 +현재 정상 동작하지만 아래 경고가 발생했음: + +```text +Direct use of automatic function calling (AFC) in Models.generate_content is not recommended... +``` + +기능에는 문제가 없으나 향후 Google 권장 API 방식으로 migration 검토. + +--- + +## 5. Groq fallback + +### [ ] Groq API 인증 문제 해결 +현재 상태: +- Groq API key를 등록했으나 401 `Invalid API Key` +- 새 key 교체 후에도 인증 문제로 일단 보류 + +향후: +- Groq Console에서 project/key 상태 재확인 +- `models.list()` 인증 테스트 +- 인증 성공 후 Gemini fallback provider로 연결 + +목표: +```text +Gemini 성공 + -> 완료 + +Gemini quota/timeout/API 오류 + -> Groq fallback + +Groq도 실패 + -> AI 없이 논문 저장 +``` + +### [ ] Groq dependency는 인증 성공 후 정식 추가 +현재 Gemini는 정식 dependency 추가 완료. +Groq는 인증 정상화 후 `pyproject.toml`에 추가. + +--- + +## 6. OA / PDF 처리 + +### [ ] Semantic Scholar `openAccessPdf` 우선 사용 검증 +계획: +1. Semantic Scholar `openAccessPdf` +2. 없으면 Unpaywall +3. 없으면 abstract fallback + +Semantic Scholar가 다시 활성화된 뒤 실제 논문에서 `pdf_url`이 정상 채워지는지 확인 필요. + +### [ ] Unpaywall 실제 신규 논문 경로 검증 +standalone 테스트는 성공했지만 실제 신규 논문 run에서 다음 로그를 아직 충분히 확인하지 못함: + +```text +Unpaywall candidates: N / M new papers +Unpaywall resolved: status=... pdf=YES ... +``` + +신규 논문 발생 시 확인. + +### [ ] PDF 다운로드 실패 유형 통계/로그 개선 +현재 예외처리는 되어 있음. + +향후 구분 로그 검토: +- 403 +- 404 +- timeout +- HTML 응답 +- invalid PDF signature +- max file size 초과 +- parse failure +- no extractable text + +### [ ] 스캔 PDF OCR 지원 여부 +현재 `pypdf` 기반 텍스트 추출. +스캔 이미지 PDF는 `No extractable text`가 될 수 있음. + +현재 권장: +- OCR은 우선 미지원 +- abstract fallback + +필요성이 커질 경우에만 OCR 기능 추가 검토. + +### [ ] PDF text 선택적 추출 개선 +현재 제한: +- 최대 40 pages +- 최대 약 120,000 chars +- 최대 PDF 25 MB + +향후 개선 검토: +- Abstract +- Introduction +- Method +- Results +- Conclusion + +섹션을 우선적으로 추출하여 Gemini에 전달. + +--- + +## 7. AI 상태 / retry 정책 + +### [ ] 실제 운영 데이터에서 `failed -> retry -> done` 경로 확인 +임시 DB E2E 테스트는 성공함. + +실제 운영에서 최초 Gemini/API 오류가 발생했을 때: +```text +ai_status = failed +``` +저장 후 다음 주 실행에서: +```text +retry -> done +``` +되는지 로그 확인. + +### [ ] `skipped` 논문 재평가 정책 결정 +현재: +```text +PDF 없음 + abstract 없음 +-> ai_status = skipped +-> 자동 재시도 안 함 +``` + +문제: +- 나중에 Semantic Scholar에서 abstract가 추가되거나 +- Unpaywall에서 OA PDF가 생길 수 있음 + +향후 정책 후보: +- skipped 논문을 30일 후 1회 재조회 +- metadata가 개선된 경우에만 재분석 +- 계속 skipped인 논문은 영구 종료 + +### [ ] AI retry 횟수 제한 추가 검토 +현재 `failed`는 이후 run에서 다시 시도 가능. + +향후 DB 필드 추가 검토: +```text +ai_retry_count +ai_last_attempt_at +``` + +예: +- 최대 3회 실패 후 `permanent_failed` +- 무한 재시도 방지 + +--- + +## 8. DB schema / 데이터 관리 + +### [ ] AI provenance 정보 추가 검토 +현재 저장: +- relevance +- categories +- summary +- ai_reason +- oa_status +- pdf_url +- ai_analysis_level +- ai_status + +추가 검토: +```text +ai_provider +ai_model +ai_processed_at +ai_retry_count +``` + +장점: +- 어떤 모델이 만든 요약인지 추적 가능 +- 모델 변경 전/후 비교 가능 + +### [ ] DB backup 정책 +`papers.db`는 시스템의 핵심 상태 데이터. + +권장: +- Synology Hyper Backup 또는 별도 주기 backup +- 최소 주 1회 +- `papers.db` + config + source code 함께 backup 검토 + +### [ ] 오래된 테스트 데이터 확인 +임시 DB는 `/tmp`에 생성했으므로 container 종료 시 영향 없음. +운영 `papers.db`에 테스트 논문이 섞여 있지 않은지 한 번 확인 권장. + +--- + +## 9. Joplin 출력 개선 + +### [ ] 실제 AI 분석된 신규 논문이 Joplin에 출력되는 것 확인 +render standalone 테스트는 성공. + +실제 신규 논문에서 확인할 항목: +```text +AI Analysis: Full text +또는 +AI Analysis: Abstract only + +AI Summary +Why relevant +OA Status +OA PDF +``` + +### [ ] Joplin note layout 개선 검토 +현재: +1. New papers 표 +2. Summaries 상세 항목 + +향후 가능: +- relevance별 section +- category별 section +- AI Full-text 분석 논문 상단 배치 +- Top 5 paper 별도 section + +### [ ] 같은 날 재실행 시 note replacement 정책 확인 +현재 같은 날짜 note를 교체하면서 해당 날짜 발견 논문을 누적하는 구조. + +주 1회 운영에서는 큰 문제 없음. + +--- + +## 10. Source 추가 + +### [ ] Google Scholar Alert 연동 +아직 보류 중. + +계획: +```text +Google Scholar Alert +-> Gmail +-> IMAP +-> ScholarGmailCollector +-> paper-monitor +``` + +웹 scraping 대신 Google Scholar Alert 메일을 사용하는 방식 권장. + +필요: +- Google Scholar Alert 생성 +- Gmail App Password +- NAS `.env` +- Gmail collector 활성화 +- sender/subject filter 테스트 + +### [ ] IEEE 복구 후 source 우선순위 재정의 +장기 목표 예: +```text +IEEE Xplore +Semantic Scholar +Crossref +Google Scholar Alert +``` + +각 source의 역할 중복을 보고 일부 source를 보조용으로 낮출지 검토. + +--- + +## 11. 보안 + +### [ ] `.env` 권한 확인 +API key, Gmail password 등이 들어있으므로 NAS에서 접근 권한 최소화. + +예: +```sh +chmod 600 .env +``` + +단, Docker/실행 사용자가 읽을 수 있는지 확인 후 적용. + +### [ ] 로그에 secret 노출 여부 재점검 +특히 확인: +- IEEE API key +- Gemini API key +- Semantic Scholar API key +- Gmail App Password + +로그에는 key value를 출력하지 않도록 유지. + +### [ ] API key rotation 정책 +키가 채팅/로그에 노출되었다면 즉시 교체. +장기적으로 필요 시 주기적 rotation 검토. + +--- + +## 12. Docker / 유지보수 + +### [ ] dependency 버전 pinning 검토 +현재 패키지 버전을 느슨하게 두면 향후 rebuild 시 API가 깨질 수 있음. + +특히: +- `google-genai` +- `pypdf` +- `requests` +- `PyYAML` + +운영 안정화 후 known-good version으로 pinning 검토. + +### [ ] Docker image rebuild 정책 +기억할 규칙: + +```text +config.nas.yaml 변경 +-> rebuild 불필요 + +.env 변경 +-> rebuild 불필요 + +Python source 변경 +-> rebuild 필요 + +pyproject.toml 변경 +-> rebuild 필요 +``` + +### [ ] 테스트 파일 유지 여부 +현재 생성한 테스트 모듈: +- `test_e2e_ai.py` +- `test_retry_ai.py` + +향후: +- 유지해서 regression test로 사용할지 +- 별도 `tests/` 폴더로 이동할지 결정 + +--- + +# 현재 운영 권장 설정 + +현재 접근 제한 상황을 고려하면 당분간 다음 구성이 안전하다. + +```yaml +app: + timezone: Asia/Seoul + lookback_days: 14 + max_papers_per_source_per_query: 20 + min_relevance: 2 + +sources: + ieee: + enabled: false + + semantic_scholar: + enabled: false + +ai: + enabled: true + provider: gemini + api_key_env: GEMINI_API_KEY + model: gemini-3.6-flash + max_papers_per_run: 20 + +oa: + enabled: true + provider: unpaywall + email_env: UNPAYWALL_EMAIL +``` + +스케줄: + +```text +매주 월요일 09:00 KST +lookback: 14일 +``` + +현재 IEEE와 Semantic Scholar가 비활성화되어 있으므로 당분간 Crossref 중심으로 수집되고, DOI가 있는 신규 논문에 대해 Unpaywall OA 검색 및 Gemini 분석이 수행된다. + +--- + +# 우선순위 요약 + +## P0 - 운영 전에 확인 +- [ ] Semantic Scholar `enabled: false` +- [ ] IEEE `enabled: false` +- [ ] `.env` secret 권한 및 IEEE key rotation 여부 확인 + +## P1 - 접근 복구 후 +- [ ] Semantic Scholar 재활성화 + 429 상태 확인 +- [ ] IEEE 웹사이트 403 해제 확인 +- [ ] IEEE API 승인 상태 확인 +- [ ] IEEE 1-query 테스트 후 재활성화 + +## P2 - 기능 보강 +- [ ] Google Scholar Alert + Gmail collector +- [ ] Groq 인증 해결 + Gemini fallback +- [ ] `skipped` 재평가 정책 +- [ ] AI retry count / provider / model / processed_at DB 저장 +- [ ] PDF section-aware extraction + +## P3 - 장기 운영 +- [ ] DB backup +- [ ] scheduler timestamp 로그 보관기간/정리 정책(예: 8주) +- [ ] dependency version pinning +- [ ] query/source 최적화 +- [ ] Joplin report layout 개선 + +--- + +# 완료 항목 (2026-08-15 기준) + +## 스케줄러 / 운영 +- [x] DSM Task Scheduler 최종 등록 확인 +- [x] DSM 월요일 09:00 scheduler 최종 등록/활성화 +- [x] DSM에서 등록한 작업 수동 1회 실행 및 `RESULT=0` 확인 +- [x] Joplin import / WebDAV sync 정상 동작 확인 +- [x] Scheduler 로그를 실행별 개별 파일로 생성하도록 적용 + +적용 로그 파일명 형식: +```text +scheduler-YYYY-MM-DD_HH-MM-SS.log +``` + +## Gemini 운영값 +- [x] `max_papers_per_run` 운영값 확정: `20` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f96f18f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "paper-monitor" +version = "1.0.0" +requires-python = ">=3.10" +dependencies = [ + "requests>=2.32,<3", + "PyYAML>=6.0,<7", + "python-dateutil>=2.9,<3", + "beautifulsoup4>=4.12,<5", + "openai>=1.0", + "tzdata", + "google-genai", + "pypdf", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +pythonpath = ["src"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c2882a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +requests>=2.32,<3 +PyYAML>=6.0,<7 +python-dateutil>=2.9,<3 +beautifulsoup4>=4.12,<5 +openai>=1.0 + +tzdata diff --git a/run_scheduled.sh b/run_scheduled.sh new file mode 100644 index 0000000..c0422e0 --- /dev/null +++ b/run_scheduled.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +PROJECT_DIR="/volume1/docker/paper-monitor" +LOG_DIR="$PROJECT_DIR/data/logs" +LOG_STAMP="$(date '+%Y-%m-%d_%H-%M-%S')" +LOG_FILE="$LOG_DIR/scheduler-${LOG_STAMP}.log" + +mkdir -p "$LOG_DIR" + +echo "============================================================" >> "$LOG_FILE" +echo "START $(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE" + +cd "$PROJECT_DIR" || exit 1 + +/usr/local/bin/docker compose \ + -f docker-compose.nas.yml \ + run --rm paper-monitor-nas \ + >> "$LOG_FILE" 2>&1 + +RESULT=$? + +echo "END $(date '+%Y-%m-%d %H:%M:%S') RESULT=$RESULT" >> "$LOG_FILE" +echo "============================================================" >> "$LOG_FILE" + +exit "$RESULT" \ No newline at end of file diff --git a/src/paper_monitor/__init__.py b/src/paper_monitor/__init__.py new file mode 100644 index 0000000..5becc17 --- /dev/null +++ b/src/paper_monitor/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0.0" diff --git a/src/paper_monitor/__main__.py b/src/paper_monitor/__main__.py new file mode 100644 index 0000000..25d26fd --- /dev/null +++ b/src/paper_monitor/__main__.py @@ -0,0 +1,3 @@ +from .runner import cli +if __name__ == "__main__": + cli() diff --git a/src/paper_monitor/ai_enrich.py b/src/paper_monitor/ai_enrich.py new file mode 100644 index 0000000..95ef5fc --- /dev/null +++ b/src/paper_monitor/ai_enrich.py @@ -0,0 +1,221 @@ +import json + + +SYSTEM = '''You are a technical literature triage assistant for an automotive radar engineer. + +Analyze ONLY the supplied metadata. +Do not invent experimental results, methods, conclusions, or claims that are not supported by the supplied metadata. + +Return: +- relevance: integer from 1 to 5 +- categories: technical categories +- summary: 1-2 concise Korean sentences +- reason: concise Korean explanation of why this paper is relevant + +Preferred categories: +Antenna, +Waveform, +DOA, +MIMO, +Target Modeling, +Interference, +RF Impairment, +Imaging Radar, +Signal Processing, +Hardware, +Other. +''' + + +AI_RESPONSE_SCHEMA = { + "type": "object", + "properties": { + "relevance": { + "type": "integer", + "minimum": 1, + "maximum": 5, + }, + "categories": { + "type": "array", + "items": { + "type": "string" + }, + }, + "summary": { + "type": "string" + }, + "reason": { + "type": "string" + }, + }, + "required": [ + "relevance", + "categories", + "summary", + "reason", + ], +} + + +def _make_payload(paper): + return { + "title": paper.title, + "abstract": paper.abstract, + "venue": paper.venue, + "year": paper.year, + "current_categories": paper.categories, + "current_relevance": paper.relevance, + } + + +def _apply_result(paper, data): + paper.relevance = max( + 1, + min( + 5, + int(data.get("relevance", paper.relevance)) + ) + ) + + if isinstance(data.get("categories"), list) and data["categories"]: + paper.categories = [ + str(x) + for x in data["categories"][:5] + ] + + paper.summary = str( + data.get("summary", "") + ).strip() + + paper.ai_reason = str( + data.get("reason", "") + ).strip() + + return paper + + +def enrich_with_gemini(paper, api_key, model): + if not api_key: + return paper + + from google import genai + from google.genai import types + + client = genai.Client(api_key=api_key) + + payload = _make_payload(paper) + + prompt = ( + SYSTEM + + "\n\nPaper metadata:\n" + + json.dumps(payload, ensure_ascii=False) + ) + + response = client.models.generate_content( + model=model, + contents=prompt, + config=types.GenerateContentConfig( + temperature=0.1, + response_mime_type="application/json", + response_schema=AI_RESPONSE_SCHEMA, + ), + ) + + data = json.loads(response.text) + + paper = _apply_result(paper, data) + paper.ai_analysis_level = "abstract" + + return paper + +def enrich_with_gemini_full_text( + paper, + full_text, + api_key, + model, +): + if not api_key: + return paper + + if not full_text or not full_text.strip(): + return paper + + from google import genai + from google.genai import types + + client = genai.Client(api_key=api_key) + + payload = { + "title": paper.title, + "abstract": paper.abstract, + "venue": paper.venue, + "year": paper.year, + "full_text": full_text, + "current_categories": paper.categories, + "current_relevance": paper.relevance, + } + + prompt = ( + SYSTEM + + """ + +You are given extracted full text from the paper. + +Base the analysis primarily on the supplied full text. +Do not invent information that is not present in the supplied text. +If the extracted text appears incomplete or corrupted, be conservative. + +The summary should describe the paper's main technical approach, +contribution, and relevance to automotive radar when supported by the text. +""" + + "\n\nPaper data:\n" + + json.dumps(payload, ensure_ascii=False) + ) + + response = client.models.generate_content( + model=model, + contents=prompt, + config=types.GenerateContentConfig( + temperature=0.1, + response_mime_type="application/json", + response_schema=AI_RESPONSE_SCHEMA, + ), + ) + + data = json.loads(response.text) + + paper = _apply_result(paper, data) + paper.ai_analysis_level = "full_text" + + return paper + +def enrich_with_openai(paper, api_key, model): + if not api_key: + return paper + + from openai import OpenAI + + client = OpenAI(api_key=api_key) + + payload = _make_payload(paper) + + response = client.responses.create( + model=model, + instructions=SYSTEM, + input=json.dumps( + payload, + ensure_ascii=False + ), + ) + + text = response.output_text.strip() + + if text.startswith("```"): + text = text.strip("`") + + if text.lower().startswith("json"): + text = text[4:].strip() + + data = json.loads(text) + + return _apply_result(paper, data) \ No newline at end of file diff --git a/src/paper_monitor/circuit_breaker.py b/src/paper_monitor/circuit_breaker.py new file mode 100644 index 0000000..dc5f94f --- /dev/null +++ b/src/paper_monitor/circuit_breaker.py @@ -0,0 +1,386 @@ +from __future__ import annotations + +import json +import logging +import os +import re +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import requests + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ProviderPolicy: + # 동일 오류가 반복될 때 적용할 cooldown 시간(초) + status_cooldowns: dict[int, tuple[int, ...]] + network_cooldowns: tuple[int, ...] | None = None + + +PROVIDER_POLICIES: dict[str, ProviderPolicy] = { + "ieee": ProviderPolicy( + status_cooldowns={ + # 1회 403: 30분 + # 2회 403: 6시간 + # 3회 이상: 24시간 + 403: (30 * 60, 6 * 60 * 60, 24 * 60 * 60), + + # 혹시 rate limit이 발생할 경우 + 429: (2 * 60, 10 * 60, 60 * 60), + + # 서버 장애 + 500: (60, 5 * 60, 30 * 60), + 502: (60, 5 * 60, 30 * 60), + 503: (60, 5 * 60, 30 * 60), + 504: (60, 5 * 60, 30 * 60), + }, + network_cooldowns=(60, 5 * 60, 30 * 60), + ), + + "semantic_scholar": ProviderPolicy( + status_cooldowns={ + # CloudFront / WAF 차단 + 403: (30 * 60, 6 * 60 * 60, 24 * 60 * 60), + + # API rate limit + 429: (2 * 60, 10 * 60, 60 * 60), + + 500: (60, 5 * 60, 30 * 60), + 502: (60, 5 * 60, 30 * 60), + 503: (60, 5 * 60, 30 * 60), + 504: (60, 5 * 60, 30 * 60), + }, + network_cooldowns=(60, 5 * 60, 30 * 60), + ), + + "semanticscholar": ProviderPolicy( + status_cooldowns={ + 403: (30 * 60, 6 * 60 * 60, 24 * 60 * 60), + 429: (2 * 60, 10 * 60, 60 * 60), + 500: (60, 5 * 60, 30 * 60), + 502: (60, 5 * 60, 30 * 60), + 503: (60, 5 * 60, 30 * 60), + 504: (60, 5 * 60, 30 * 60), + }, + network_cooldowns=(60, 5 * 60, 30 * 60), + ), + + "crossref": ProviderPolicy( + status_cooldowns={ + 429: (2 * 60, 10 * 60, 60 * 60), + 500: (60, 5 * 60, 30 * 60), + 502: (60, 5 * 60, 30 * 60), + 503: (60, 5 * 60, 30 * 60), + 504: (60, 5 * 60, 30 * 60), + }, + network_cooldowns=(60, 5 * 60, 30 * 60), + ), +} + + +class CircuitBreaker: + def __init__(self, state_file: str | Path): + self.state_file = Path(state_file) + self.state_file.parent.mkdir(parents=True, exist_ok=True) + self.state = self._load() + + def _load(self) -> dict[str, Any]: + if not self.state_file.exists(): + return {} + + try: + with self.state_file.open("r", encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, dict): + return data + + except Exception as exc: + logger.warning( + "Failed to load circuit breaker state: %s", + exc, + ) + + return {} + + def _save(self) -> None: + """ + 임시 파일 작성 후 atomic replace. + 저장 중 프로세스가 죽어도 JSON 파일이 깨질 가능성을 낮춘다. + """ + tmp_file = self.state_file.with_suffix( + self.state_file.suffix + ".tmp" + ) + + with tmp_file.open("w", encoding="utf-8") as f: + json.dump( + self.state, + f, + indent=2, + ensure_ascii=False, + ) + + os.replace(tmp_file, self.state_file) + + def _provider_state(self, provider: str) -> dict[str, Any]: + if provider not in self.state: + self.state[provider] = { + "open_until": 0.0, + "reason": None, + "strikes": {}, + } + + return self.state[provider] + + def is_open(self, provider: str) -> bool: + state = self._provider_state(provider) + + open_until = float(state.get("open_until", 0.0)) + + return time.time() < open_until + + def remaining_seconds(self, provider: str) -> int: + state = self._provider_state(provider) + + remaining = float(state.get("open_until", 0.0)) - time.time() + + return max(0, int(remaining)) + + def reason(self, provider: str) -> str | None: + return self._provider_state(provider).get("reason") + + def record_success(self, provider: str) -> None: + """ + 실제 provider 요청이 성공했으면 이전 failure strike를 초기화한다. + """ + old_state = self._provider_state(provider) + + had_failure = ( + old_state.get("strikes") + or old_state.get("open_until", 0.0) > 0 + ) + + self.state[provider] = { + "open_until": 0.0, + "reason": None, + "strikes": {}, + } + + if had_failure: + self._save() + logger.info( + "[CircuitBreaker] %s recovered; state reset", + provider, + ) + + def record_failure( + self, + provider: str, + failure_key: str, + cooldowns: tuple[int, ...], + ) -> int: + state = self._provider_state(provider) + + strikes = state.setdefault("strikes", {}) + + strike = int(strikes.get(failure_key, 0)) + 1 + strikes[failure_key] = strike + + cooldown_index = min( + strike - 1, + len(cooldowns) - 1, + ) + + cooldown = cooldowns[cooldown_index] + + state["open_until"] = time.time() + cooldown + state["reason"] = failure_key + + self._save() + + logger.warning( + "[CircuitBreaker] OPEN provider=%s reason=%s " + "strike=%d cooldown=%ds", + provider, + failure_key, + strike, + cooldown, + ) + + return cooldown + + +def get_provider_name(collector: Any) -> str: + """ + 예: + paper_monitor.collectors.ieee + -> ieee + + paper_monitor.collectors.semantic_scholar + -> semantic_scholar + """ + module_name = collector.__class__.__module__ + + return module_name.rsplit(".", 1)[-1].lower() + + +def extract_http_status(exc: Exception) -> int | None: + """ + requests.HTTPError뿐 아니라 IEEEAPIError 같은 custom exception도 + message 안에 403 등이 있으면 탐지한다. + """ + + # requests.HTTPError + if isinstance(exc, requests.HTTPError): + response = getattr(exc, "response", None) + + if response is not None: + return response.status_code + + # custom exception이 status_code를 갖는 경우 + status = getattr(exc, "status_code", None) + + if isinstance(status, int): + return status + + # custom exception 내부에 response가 있는 경우 + response = getattr(exc, "response", None) + + if response is not None: + status = getattr(response, "status_code", None) + + if isinstance(status, int): + return status + + # 마지막 fallback: + # "HTTP 403", "403 Forbidden", "status=403" 등 + text = str(exc) + + match = re.search( + r"\b(400|401|403|404|408|409|425|429|500|502|503|504)\b", + text, + ) + + if match: + return int(match.group(1)) + + return None + + +def is_network_error(exc: Exception) -> bool: + return isinstance( + exc, + ( + requests.Timeout, + requests.ConnectionError, + ), + ) + + +STATE_FILE = os.getenv( + "CIRCUIT_BREAKER_STATE", + "/state/provider_circuit_breakers.json", +) + +breaker = CircuitBreaker(STATE_FILE) + + +def protected_search(collector: Any, query: str): + provider = get_provider_name(collector) + + policy = PROVIDER_POLICIES.get(provider) + + # 별도 정책이 없는 collector는 기존 방식 그대로 + if policy is None: + return collector.search(query) + + # ============================================ + # Circuit OPEN + # ============================================ + + if breaker.is_open(provider): + remaining = breaker.remaining_seconds(provider) + reason = breaker.reason(provider) + + logger.warning( + "[CircuitBreaker] SKIP provider=%s " + "reason=%s remaining=%ds", + provider, + reason, + remaining, + ) + + return [] + + # ============================================ + # 실제 provider 호출 + # ============================================ + + try: + result = collector.search(query) + + except Exception as exc: + status = extract_http_status(exc) + + # ---------------------------------------- + # HTTP failure + # ---------------------------------------- + + if status is not None: + cooldowns = policy.status_cooldowns.get(status) + + if cooldowns: + breaker.record_failure( + provider, + f"http_{status}", + cooldowns, + ) + + logger.warning( + "[%s] request failed with HTTP %s; " + "provider disabled temporarily", + provider, + status, + ) + + return [] + + # ---------------------------------------- + # Network failure + # ---------------------------------------- + + if ( + is_network_error(exc) + and policy.network_cooldowns is not None + ): + breaker.record_failure( + provider, + "network_error", + policy.network_cooldowns, + ) + + logger.warning( + "[%s] network error; " + "provider disabled temporarily: %s", + provider, + exc, + ) + + return [] + + # circuit breaker 대상이 아닌 exception은 + # 기존 코드에서 처리할 수 있도록 다시 raise + raise + + # ============================================ + # 성공 + # ============================================ + + breaker.record_success(provider) + + return result \ No newline at end of file diff --git a/src/paper_monitor/collectors/__init__.py b/src/paper_monitor/collectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/paper_monitor/collectors/crossref.py b/src/paper_monitor/collectors/crossref.py new file mode 100644 index 0000000..2b2ef45 --- /dev/null +++ b/src/paper_monitor/collectors/crossref.py @@ -0,0 +1,26 @@ +from datetime import datetime,timedelta,timezone +import requests +from ..models import Paper +from ..util import clean_text,normalize_doi,parse_date + +class CrossrefCollector: + def __init__(self,base_url,mailto,lookback_days,limit): + self.base_url,self.mailto,self.lookback_days,self.limit=base_url,mailto,lookback_days,min(limit,100) + def search(self,query): + since=(datetime.now(timezone.utc).date()-timedelta(days=self.lookback_days)).isoformat() + params={'query.bibliographic':query,'filter':f'from-pub-date:{since}','rows':self.limit,'sort':'published','order':'desc'} + if self.mailto: params['mailto']=self.mailto + headers={'User-Agent':f'paper-monitor/1.0 (mailto:{self.mailto})' if self.mailto else 'paper-monitor/1.0'} + r=requests.get(self.base_url,params=params,headers=headers,timeout=(10, 30)); r.raise_for_status() + out=[] + for a in r.json().get('message',{}).get('items',[]): + title=clean_text((a.get('title') or [''])[0]); + if not title: continue + authors=[] + for x in a.get('author',[]): + name=' '.join(filter(None,[x.get('given'),x.get('family')])).strip() + if name: authors.append(name) + date=parse_date(a.get('published-print') or a.get('published-online') or a.get('published')) + doi=normalize_doi(a.get('DOI')); container=a.get('container-title') or [] + out.append(Paper(title=title,authors=authors,abstract=clean_text(a.get('abstract')),doi=doi,url=a.get('URL') or (f'https://doi.org/{doi}' if doi else ''),venue=clean_text(container[0] if container else ''),publication_date=date,year=int(date[:4]) if len(date)>=4 and date[:4].isdigit() else None,source='Crossref',source_id=doi)) + return out diff --git a/src/paper_monitor/collectors/ieee.py b/src/paper_monitor/collectors/ieee.py new file mode 100644 index 0000000..0fd2ec2 --- /dev/null +++ b/src/paper_monitor/collectors/ieee.py @@ -0,0 +1,249 @@ +import time + +import requests + +from ..models import Paper +from ..util import ( + clean_text, + normalize_doi, + parse_date, + within_lookback, +) + + +class IEEEAPIError(RuntimeError): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class IEEECollector: + def __init__( + self, + base_url, + api_key, + lookback_days, + limit, + min_request_interval=5.0, + ): + self.base_url = base_url + self.api_key = api_key + self.lookback_days = lookback_days + self.limit = limit + + # Conservative client-side pacing. + # This is NOT an IEEE-defined fixed rate limit. + self.min_request_interval = min_request_interval + self._last_request_time = 0.0 + + self.session = requests.Session() + + self.session.headers.update( + { + "Accept": "application/json", + "User-Agent": ( + "paper-monitor/1.0 " + "(academic literature monitoring)" + ), + } + ) + + def _wait_for_rate_limit(self): + elapsed = ( + time.monotonic() + - self._last_request_time + ) + + wait = ( + self.min_request_interval + - elapsed + ) + + if wait > 0: + time.sleep(wait) + + def _request(self, params): + self._wait_for_rate_limit() + + try: + response = self.session.get( + self.base_url, + params=params, + timeout=(10, 30), + ) + + self._last_request_time = ( + time.monotonic() + ) + + except requests.exceptions.Timeout: + self._last_request_time = ( + time.monotonic() + ) + + # Do not expose the original requests exception, + # because its URL may contain the API key. + raise IEEEAPIError( + None, + "IEEE API request timed out", + ) from None + + except requests.exceptions.ConnectionError: + self._last_request_time = ( + time.monotonic() + ) + + raise IEEEAPIError( + None, + "IEEE API connection failed", + ) from None + + except requests.exceptions.RequestException: + self._last_request_time = ( + time.monotonic() + ) + + raise IEEEAPIError( + None, + "IEEE API request failed", + ) from None + + status = response.status_code + + if status == 401: + raise IEEEAPIError( + 401, + "IEEE API authentication failed", + ) + + if status == 403: + raise IEEEAPIError( + 403, + "IEEE API access forbidden", + ) + + if status == 429: + retry_after = response.headers.get( + "Retry-After" + ) + + if retry_after: + raise IEEEAPIError( + 429, + ( + "IEEE API rate limited " + f"(Retry-After={retry_after})" + ), + ) + + raise IEEEAPIError( + 429, + "IEEE API rate limited", + ) + + if status >= 400: + raise IEEEAPIError( + status, + f"IEEE API HTTP error {status}", + ) + + return response + + def search(self, query): + if not self.api_key: + return [] + + params = { + "apikey": self.api_key, + "format": "json", + "max_records": min( + self.limit, + 200, + ), + "start_record": 1, + "sort_order": "desc", + "sort_field": "publication_year", + "querytext": query, + } + + response = self._request(params) + + out = [] + + for article in ( + response.json() + .get("articles", []) + ): + date = parse_date( + article.get("publication_date") + or article.get( + "publication_year" + ) + ) + + if not within_lookback( + date, + self.lookback_days, + ): + continue + + authors = [ + clean_text( + author.get("full_name") + ) + for author in ( + article.get("authors") + or {} + ).get("authors", []) + if author.get("full_name") + ] + + year_text = str( + article.get( + "publication_year", + "", + ) + ) + + year = ( + int(year_text) + if year_text.isdigit() + else None + ) + + paper = Paper( + title=clean_text( + article.get("title") + ), + authors=authors, + abstract=clean_text( + article.get("abstract") + ), + doi=normalize_doi( + article.get("doi") + ), + url=( + article.get("html_url") + or article.get("pdf_url") + or "" + ), + venue=clean_text( + article.get( + "publication_title" + ) + ), + publication_date=date, + year=year, + source="IEEE Xplore", + source_id=str( + article.get( + "article_number" + ) + or "" + ), + ) + + if paper.title: + out.append(paper) + + return out \ No newline at end of file diff --git a/src/paper_monitor/collectors/scholar_gmail.py b/src/paper_monitor/collectors/scholar_gmail.py new file mode 100644 index 0000000..62e091e --- /dev/null +++ b/src/paper_monitor/collectors/scholar_gmail.py @@ -0,0 +1,65 @@ +from datetime import datetime,timedelta,timezone +import email, imaplib, re +from email.header import decode_header, make_header +from bs4 import BeautifulSoup +from ..models import Paper +from ..util import clean_text, normalize_doi, DOI_RE + +def _decode(v): + if not v: return '' + try: return str(make_header(decode_header(v))) + except Exception: return v + +class ScholarGmailCollector: + def __init__(self,address,app_password,imap_host,mailbox,sender_contains,subject_contains,lookback_days): + self.address,self.password,self.host,self.mailbox=address,app_password,imap_host,mailbox + self.sender_contains,self.subject_contains=sender_contains.lower(),subject_contains.lower() + self.lookback_days=lookback_days + def collect(self): + if not self.address or not self.password: return [] + since=(datetime.now(timezone.utc)-timedelta(days=self.lookback_days)).strftime('%d-%b-%Y') + with imaplib.IMAP4_SSL(self.host) as m: + m.login(self.address,self.password); m.select(self.mailbox,readonly=True) + typ,data=m.search(None,'SINCE',since) + if typ!='OK': return [] + papers=[] + for num in data[0].split(): + typ,msgdata=m.fetch(num,'(RFC822)') + if typ!='OK': continue + msg=email.message_from_bytes(msgdata[0][1]) + sender,subject=_decode(msg.get('From')).lower(),_decode(msg.get('Subject')).lower() + if self.sender_contains and self.sender_contains not in sender: continue + if self.subject_contains and self.subject_contains not in subject: continue + papers.extend(self._parse_message(msg)) + return papers + def _parse_message(self,msg): + html_body=text_body='' + parts=msg.walk() if msg.is_multipart() else [msg] + for part in parts: + if 'attachment' in str(part.get('Content-Disposition','')).lower(): continue + payload=part.get_payload(decode=True) + if not payload: continue + body=payload.decode(part.get_content_charset() or 'utf-8',errors='replace') + if part.get_content_type()=='text/html': html_body+=body + elif part.get_content_type()=='text/plain': text_body+=body + date='' + try: date=email.utils.parsedate_to_datetime(msg.get('Date')).date().isoformat() + except Exception: pass + papers=[] + if html_body: + soup=BeautifulSoup(html_body,'html.parser') + for a in soup.find_all('a',href=True): + title=clean_text(a.get_text(' ',strip=True)); href=a['href'] + if len(title)<15: continue + if any(x in title.lower() for x in ['create alert','cancel alert','view all','unsubscribe']): continue + context=clean_text(a.parent.get_text(' ',strip=True) if a.parent else title) + doi_match=DOI_RE.search(context+' '+href) + papers.append(Paper(title=title,abstract=context if context!=title else '',doi=normalize_doi(doi_match.group(0) if doi_match else ''),url=href,publication_date=date,source='Google Scholar Alert')) + else: + for line in text_body.splitlines(): + m=re.search(r'(https?://\S+)',line) + if m: + title=clean_text(line[:m.start()]) + if len(title)>=15: papers.append(Paper(title=title,url=m.group(1),publication_date=date,source='Google Scholar Alert')) + uniq={p.title.lower():p for p in papers} + return list(uniq.values()) diff --git a/src/paper_monitor/collectors/semantic_scholar.py b/src/paper_monitor/collectors/semantic_scholar.py new file mode 100644 index 0000000..61349f1 --- /dev/null +++ b/src/paper_monitor/collectors/semantic_scholar.py @@ -0,0 +1,192 @@ +import random +import time + +import requests + +from ..models import Paper + + +class SemanticScholarCollector: + def __init__( + self, + api_key, + lookback_days, + limit=20, + min_request_interval=2.5, + max_retries=3, + ): + self.api_key = api_key + self.lookback_days = lookback_days + self.limit = limit + self.min_request_interval = min_request_interval + self.max_retries = max_retries + + self.base_url = ( + "https://api.semanticscholar.org/graph/v1/paper/search" + ) + + self._last_request_time = 0.0 + + def _wait_for_rate_limit(self): + now = time.monotonic() + + elapsed = now - self._last_request_time + + wait = self.min_request_interval - elapsed + + if wait > 0: + time.sleep(wait) + + def _request(self, params): + headers = {} + + if self.api_key: + headers["x-api-key"] = self.api_key + + last_error = None + + for attempt in range(self.max_retries): + self._wait_for_rate_limit() + + try: + response = requests.get( + self.base_url, + params=params, + headers=headers, + timeout=(10, 30), + ) + + self._last_request_time = time.monotonic() + + if response.status_code == 429: + retry_after = response.headers.get( + "Retry-After" + ) + + if retry_after: + try: + delay = float(retry_after) + except ValueError: + delay = None + else: + delay = None + + if delay is None: + delay = ( + 2.5 * (2 ** attempt) + + random.uniform(0.2, 0.8) + ) + + print( + "Semantic Scholar rate limited. " + f"Retrying after {delay:.1f} sec..." + ) + + time.sleep(delay) + last_error = requests.exceptions.HTTPError( + f"429 Too Many Requests for {response.url}", + response=response, + ) + + continue + + response.raise_for_status() + + return response + + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ) as exc: + self._last_request_time = time.monotonic() + + last_error = exc + + if attempt >= self.max_retries - 1: + raise + + delay = ( + 2.5 * (2 ** attempt) + + random.uniform(0.2, 0.8) + ) + + print( + "Semantic Scholar network error. " + f"Retrying after {delay:.1f} sec..." + ) + + time.sleep(delay) + + if last_error: + raise last_error + + raise RuntimeError( + "Semantic Scholar request failed" + ) + + def search(self, query): + params = { + "query": query, + "limit": self.limit, + "fields": ( + "paperId," + "title," + "abstract," + "authors," + "year," + "venue," + "publicationDate," + "citationCount," + "externalIds," + "url," + "openAccessPdf" + ), + } + + response = self._request(params) + + data = response.json() + + papers = [] + + for item in data.get("data", []): + external_ids = item.get("externalIds") or {} + + doi = external_ids.get("DOI") or "" + + authors = [ + a.get("name", "") + for a in item.get("authors", []) + if a.get("name") + ] + + open_access = item.get("openAccessPdf") or {} + + pdf_url = ( + open_access.get("url") + if isinstance(open_access, dict) + else "" + ) or "" + + paper = Paper( + title=item.get("title") or "", + authors=authors, + abstract=item.get("abstract") or "", + doi=doi, + url=item.get("url") or "", + venue=item.get("venue") or "", + publication_date=( + item.get("publicationDate") or "" + ), + year=item.get("year"), + citation_count=item.get( + "citationCount" + ), + source="Semantic Scholar", + source_id=item.get("paperId") or "", + pdf_url=pdf_url, + ) + + papers.append(paper) + + return papers \ No newline at end of file diff --git a/src/paper_monitor/config.py b/src/paper_monitor/config.py new file mode 100644 index 0000000..4e8d016 --- /dev/null +++ b/src/paper_monitor/config.py @@ -0,0 +1,16 @@ +from pathlib import Path +import os, yaml + +def load_config(path: str) -> dict: + p = Path(path).expanduser().resolve() + with p.open("r", encoding="utf-8") as f: + cfg = yaml.safe_load(f) + cfg["_config_dir"] = str(p.parent) + return cfg + +def env_value(name: str | None, default: str = "") -> str: + return os.getenv(name, default) if name else default + +def resolve_path(cfg: dict, value: str) -> str: + p = Path(value) + return str(p if p.is_absolute() else (Path(cfg["_config_dir"]) / p).resolve()) diff --git a/src/paper_monitor/db.py b/src/paper_monitor/db.py new file mode 100644 index 0000000..0687c1c --- /dev/null +++ b/src/paper_monitor/db.py @@ -0,0 +1,241 @@ +import sqlite3 +from pathlib import Path +from datetime import datetime, timezone +from .models import Paper +from .util import paper_key + +SCHEMA = ''' +CREATE TABLE IF NOT EXISTS papers ( + paper_key TEXT PRIMARY KEY, + doi TEXT, + title TEXT NOT NULL, + authors TEXT, + abstract TEXT, + url TEXT, + venue TEXT, + publication_date TEXT, + year INTEGER, + citation_count INTEGER, + source TEXT, + source_id TEXT, + categories TEXT, + relevance INTEGER DEFAULT 0, + summary TEXT, + ai_reason TEXT, + first_seen TEXT NOT NULL, + first_seen_local_date TEXT, + last_seen TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_papers_pubdate ON papers(publication_date); +CREATE INDEX IF NOT EXISTS idx_papers_relevance ON papers(relevance); +CREATE INDEX IF NOT EXISTS idx_papers_first_seen_local_date ON papers(first_seen_local_date); +''' + +class PaperDB: + def __init__(self, path): + Path(path).parent.mkdir(parents=True, exist_ok=True) + self.conn = sqlite3.connect(path) + self.conn.row_factory = sqlite3.Row + self.conn.executescript(SCHEMA) + self._migrate() + + def _migrate(self): + cols = { + r['name'] + for r in self.conn.execute( + 'PRAGMA table_info(papers)' + ).fetchall() + } + + if 'first_seen_local_date' not in cols: + self.conn.execute( + 'ALTER TABLE papers ADD COLUMN first_seen_local_date TEXT' + ) + + if 'oa_status' not in cols: + self.conn.execute( + 'ALTER TABLE papers ADD COLUMN oa_status TEXT' + ) + + if 'pdf_url' not in cols: + self.conn.execute( + 'ALTER TABLE papers ADD COLUMN pdf_url TEXT' + ) + + if 'ai_analysis_level' not in cols: + self.conn.execute( + 'ALTER TABLE papers ADD COLUMN ai_analysis_level TEXT' + ) + + if 'ai_status' not in cols: + self.conn.execute( + 'ALTER TABLE papers ADD COLUMN ai_status TEXT' + ) + + self.conn.commit() + + def exists(self, p: Paper) -> bool: + k = paper_key(p.doi, p.title) + row = self.conn.execute( + 'SELECT 1 FROM papers WHERE paper_key=? LIMIT 1', + (k,) + ).fetchone() + return row is not None + + def upsert(self, p: Paper, local_date: str = ''): + k = paper_key(p.doi, p.title) + now = datetime.now(timezone.utc).isoformat() + old = self.conn.execute('SELECT * FROM papers WHERE paper_key=?', (k,)).fetchone() + is_new = old is None + authors, cats = '|||'.join(p.authors), '|||'.join(p.categories) + if is_new: + self.conn.execute('''INSERT INTO papers + (paper_key,doi,title,authors,abstract,url,venue,publication_date,year,citation_count,source,source_id,categories,relevance,summary,ai_reason,oa_status,pdf_url,ai_analysis_level,ai_status,first_seen,first_seen_local_date,last_seen) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', + ( + k, + p.doi, + p.title, + authors, + p.abstract, + p.url, + p.venue, + p.publication_date, + p.year, + p.citation_count, + p.source, + p.source_id, + cats, + p.relevance, + p.summary, + p.ai_reason, + p.oa_status, + p.pdf_url, + p.ai_analysis_level, + p.ai_status, + now, + local_date, + now, + )) + else: + def choose(new, oldv): return new if new not in ('', None, [], 0) else oldv + self.conn.execute('''UPDATE papers SET + doi=?, + title=?, + authors=?, + abstract=?, + url=?, + venue=?, + publication_date=?, + year=?, + citation_count=?, + source=?, + source_id=?, + categories=?, + relevance=?, + summary=?, + ai_reason=?, + oa_status=?, + pdf_url=?, + ai_analysis_level=?, + ai_status=?, + last_seen=? + WHERE paper_key=?''', + ( + choose(p.doi, old['doi']), + choose(p.title, old['title']), + choose(authors, old['authors']), + choose(p.abstract, old['abstract']), + choose(p.url, old['url']), + choose(p.venue, old['venue']), + choose(p.publication_date, old['publication_date']), + choose(p.year, old['year']), + choose(p.citation_count, old['citation_count']), + choose(p.source, old['source']), + choose(p.source_id, old['source_id']), + choose(cats, old['categories']), + max(p.relevance, old['relevance'] or 0), + choose(p.summary, old['summary']), + choose(p.ai_reason, old['ai_reason']), + choose(p.oa_status, old['oa_status']), + choose(p.pdf_url, old['pdf_url']), + choose(p.ai_analysis_level, old['ai_analysis_level']), + choose(p.ai_status, old['ai_status']), + now, + k, + )) + self.conn.commit() + return is_new + + def list_first_seen_on(self, local_date: str) -> list[Paper]: + rows = self.conn.execute('SELECT * FROM papers WHERE first_seen_local_date=? ORDER BY relevance DESC, publication_date DESC', (local_date,)).fetchall() + out = [] + for r in rows: + out.append(Paper( + title=r['title'], + authors=(r['authors'] or '').split('|||') if r['authors'] else [], + abstract=r['abstract'] or '', + doi=r['doi'] or '', + url=r['url'] or '', + venue=r['venue'] or '', + publication_date=r['publication_date'] or '', + year=r['year'], + citation_count=r['citation_count'], + source=r['source'] or '', + source_id=r['source_id'] or '', + categories=(r['categories'] or '').split('|||') if r['categories'] else [], + relevance=r['relevance'] or 0, + summary=r['summary'] or '', + ai_reason=r['ai_reason'] or '', + oa_status=r['oa_status'] or '', + pdf_url=r['pdf_url'] or '', + ai_analysis_level=r['ai_analysis_level'] or '', + ai_status=r['ai_status'] or '', + )) + return out + + def list_ai_failed(self, limit: int = 20) -> list[Paper]: + rows = self.conn.execute( + ''' + SELECT * + FROM papers + WHERE ai_status='failed' + ORDER BY last_seen DESC + LIMIT ? + ''', + (limit,) + ).fetchall() + + out = [] + + for r in rows: + out.append( + Paper( + title=r['title'], + authors=(r['authors'] or '').split('|||') + if r['authors'] else [], + abstract=r['abstract'] or '', + doi=r['doi'] or '', + url=r['url'] or '', + venue=r['venue'] or '', + publication_date=r['publication_date'] or '', + year=r['year'], + citation_count=r['citation_count'], + source=r['source'] or '', + source_id=r['source_id'] or '', + categories=(r['categories'] or '').split('|||') + if r['categories'] else [], + relevance=r['relevance'] or 0, + summary=r['summary'] or '', + ai_reason=r['ai_reason'] or '', + oa_status=r['oa_status'] or '', + pdf_url=r['pdf_url'] or '', + ai_analysis_level=r['ai_analysis_level'] or '', + ai_status=r['ai_status'] or '', + ) + ) + + return out + + def close(self): + self.conn.close() diff --git a/src/paper_monitor/joplin.py b/src/paper_monitor/joplin.py new file mode 100644 index 0000000..f92e912 --- /dev/null +++ b/src/paper_monitor/joplin.py @@ -0,0 +1,41 @@ +import requests +class JoplinClient: + def __init__(self,base_url,token): self.base=base_url.rstrip('/'); self.token=token; self.session=requests.Session() + def _params(self,**kwargs): return {'token':self.token,**kwargs} + def ping(self): + r=self.session.get(f'{self.base}/ping',params=self._params(),timeout=10); r.raise_for_status(); return r.text + def list_folders(self,page=1): + r=self.session.get(f'{self.base}/folders',params=self._params(fields='id,title,parent_id',limit=100,page=page),timeout=15); r.raise_for_status(); return r.json() + def ensure_folder_path(self,titles): + parent_id='' + for title in titles: + found=None; page=1 + while True: + data=self.list_folders(page) + found=next((f for f in data.get('items',[]) if f.get('title')==title and (f.get('parent_id') or '')==parent_id),None) + if found or not data.get('has_more'): break + page+=1 + if not found: + r=self.session.post(f'{self.base}/folders',params=self._params(),json={'title':title,'parent_id':parent_id},timeout=15); r.raise_for_status(); found=r.json() + parent_id=found['id'] + return parent_id + def find_note_by_title(self,title,parent_id): + page=1 + while True: + r=self.session.get(f'{self.base}/notes',params=self._params(fields='id,title,parent_id',parent_id=parent_id,limit=100,page=page),timeout=15); r.raise_for_status(); data=r.json() + n=next((x for x in data.get('items',[]) if x.get('title')==title),None) + if n or not data.get('has_more'): return n + page+=1 + def create_or_update_note(self,title,body,parent_id,update=True): + old=self.find_note_by_title(title,parent_id) if update else None; payload={'title':title,'body':body,'parent_id':parent_id} + if old: r=self.session.put(f'{self.base}/notes/{old["id"]}',params=self._params(),json=payload,timeout=30) + else: r=self.session.post(f'{self.base}/notes',params=self._params(),json=payload,timeout=30) + r.raise_for_status(); return r.json()['id'] + def ensure_tag(self,title): + r=self.session.get(f'{self.base}/tags',params=self._params(fields='id,title',limit=100),timeout=15); r.raise_for_status() + old=next((x for x in r.json().get('items',[]) if x.get('title')==title),None) + if old: return old['id'] + r=self.session.post(f'{self.base}/tags',params=self._params(),json={'title':title},timeout=15); r.raise_for_status(); return r.json()['id'] + def add_tag_to_note(self,tag_id,note_id): + r=self.session.post(f'{self.base}/tags/{tag_id}/notes',params=self._params(),json={'id':note_id},timeout=15) + if r.status_code not in (200,201,204): r.raise_for_status() diff --git a/src/paper_monitor/models.py b/src/paper_monitor/models.py new file mode 100644 index 0000000..3afcf6f --- /dev/null +++ b/src/paper_monitor/models.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass, field +from typing import Optional + +@dataclass +class Paper: + title: str + authors: list[str] = field(default_factory=list) + abstract: str = "" + doi: str = "" + url: str = "" + venue: str = "" + publication_date: str = "" + year: Optional[int] = None + citation_count: Optional[int] = None + source: str = "" + source_id: str = "" + categories: list[str] = field(default_factory=list) + relevance: int = 0 + summary: str = "" + ai_reason: str = "" + oa_status: str = "" + pdf_url: str = "" + ai_analysis_level: str = "" + ai_status: str = "" + + @property + def author_text(self) -> str: + if len(self.authors) <= 4: + return ", ".join(self.authors) + return ", ".join(self.authors[:3]) + " et al." diff --git a/src/paper_monitor/oa_resolver.py b/src/paper_monitor/oa_resolver.py new file mode 100644 index 0000000..9929afb --- /dev/null +++ b/src/paper_monitor/oa_resolver.py @@ -0,0 +1,60 @@ +import logging + +import requests + + +log = logging.getLogger(__name__) + + +class UnpaywallResolver: + def __init__(self, email: str, timeout=(10, 30)): + self.email = (email or "").strip() + self.timeout = timeout + self.base_url = "https://api.unpaywall.org/v2" + + def resolve(self, paper): + if not paper.doi: + return paper + + if not self.email: + log.warning( + "Unpaywall email missing; OA lookup skipped: %s", + paper.title, + ) + return paper + + doi = paper.doi.strip() + + try: + response = requests.get( + f"{self.base_url}/{doi}", + params={"email": self.email}, + timeout=self.timeout, + ) + response.raise_for_status() + + except requests.exceptions.HTTPError as exc: + status = exc.response.status_code if exc.response else None + + # DOI not present in Unpaywall is not fatal. + if status == 404: + paper.oa_status = "not_found" + return paper + + raise + + data = response.json() + + paper.oa_status = str( + data.get("oa_status") or "" + ) + + best = data.get("best_oa_location") or {} + + pdf_url = best.get("url_for_pdf") or "" + + # Some OA records do not expose url_for_pdf but do expose a landing URL. + # We intentionally do NOT treat url as a PDF here. + paper.pdf_url = str(pdf_url).strip() + + return paper \ No newline at end of file diff --git a/src/paper_monitor/pdf_utils.py b/src/paper_monitor/pdf_utils.py new file mode 100644 index 0000000..9ae52a9 --- /dev/null +++ b/src/paper_monitor/pdf_utils.py @@ -0,0 +1,169 @@ +from io import BytesIO + +import requests +from pypdf import PdfReader + + +class PDFDownloadError(RuntimeError): + pass + + +class PDFParseError(RuntimeError): + pass + + +def download_pdf( + url: str, + timeout=(10, 45), + max_bytes: int = 25 * 1024 * 1024, +) -> bytes: + if not url: + raise PDFDownloadError("PDF URL is empty") + + headers = { + "User-Agent": ( + "paper-monitor/1.0 " + "(automated academic literature monitoring)" + ) + } + + response = requests.get( + url, + headers=headers, + timeout=timeout, + stream=True, + allow_redirects=True, + ) + + response.raise_for_status() + + content_type = ( + response.headers.get("Content-Type", "") + .lower() + .split(";")[0] + .strip() + ) + + content_length = response.headers.get("Content-Length") + + if content_length: + try: + declared_size = int(content_length) + + if declared_size > max_bytes: + raise PDFDownloadError( + f"PDF too large: {declared_size} bytes" + ) + except ValueError: + pass + + data = bytearray() + + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + + data.extend(chunk) + + if len(data) > max_bytes: + raise PDFDownloadError( + f"PDF exceeded maximum size: {max_bytes} bytes" + ) + + raw = bytes(data) + + if not raw: + raise PDFDownloadError("Downloaded PDF is empty") + + # Some OA servers return application/octet-stream, + # so do not rely only on Content-Type. + is_pdf_content_type = content_type == "application/pdf" + has_pdf_signature = raw[:5] == b"%PDF-" + + if not is_pdf_content_type and not has_pdf_signature: + raise PDFDownloadError( + f"Downloaded content is not a PDF " + f"(Content-Type={content_type or 'unknown'})" + ) + + if not has_pdf_signature: + raise PDFDownloadError( + "Downloaded content does not have a valid PDF signature" + ) + + return raw + + +def extract_pdf_text( + pdf_data: bytes, + max_pages: int = 40, + max_chars: int = 120000, +) -> str: + if not pdf_data: + raise PDFParseError("PDF data is empty") + + try: + reader = PdfReader( + BytesIO(pdf_data), + strict=False, + ) + except Exception as exc: + raise PDFParseError( + f"Failed to open PDF: {exc}" + ) from exc + + texts = [] + + page_count = min( + len(reader.pages), + max_pages, + ) + + for page_index in range(page_count): + try: + text = reader.pages[page_index].extract_text() or "" + except Exception: + # One bad page should not make the whole document unusable. + continue + + text = text.strip() + + if not text: + continue + + texts.append(text) + + if sum(len(x) for x in texts) >= max_chars: + break + + result = "\n\n".join(texts).strip() + + if not result: + raise PDFParseError( + "No extractable text found in PDF" + ) + + if len(result) > max_chars: + result = result[:max_chars] + + return result + + +def download_and_extract_pdf( + url: str, + timeout=(10, 45), + max_bytes: int = 25 * 1024 * 1024, + max_pages: int = 40, + max_chars: int = 120000, +) -> str: + pdf_data = download_pdf( + url, + timeout=timeout, + max_bytes=max_bytes, + ) + + return extract_pdf_text( + pdf_data, + max_pages=max_pages, + max_chars=max_chars, + ) \ No newline at end of file diff --git a/src/paper_monitor/render.py b/src/paper_monitor/render.py new file mode 100644 index 0000000..efa28b3 --- /dev/null +++ b/src/paper_monitor/render.py @@ -0,0 +1,76 @@ +from pathlib import Path +from .util import md_escape,safe_url + +def stars(n): return '★'*max(1,min(5,int(n or 1))) + +def render_markdown(papers,title,generated_at): + papers=sorted(papers,key=lambda p:(p.relevance,p.publication_date or '',p.citation_count or 0),reverse=True) + lines=[f'# {title}','',f'- Generated: {generated_at.isoformat(timespec="minutes")}',f'- Papers: {len(papers)}','','## New papers','','| Relevance | Category | Paper | Authors | Venue | Date | Citations | Source |','|---|---|---|---|---|---|---:|---|'] + for p in papers: + link=f'[{md_escape(p.title)}]({safe_url(p.url)})' if p.url else md_escape(p.title) + lines.append('| '+' | '.join([stars(p.relevance),md_escape(', '.join(p.categories)),link,md_escape(p.author_text),md_escape(p.venue),md_escape(p.publication_date),str(p.citation_count if p.citation_count is not None else ''),md_escape(p.source)])+' |') + if papers: + lines+=['','## Summaries',''] + for p in papers: + lines += [ + f'### {p.title}', + '', + f'- **Relevance:** {stars(p.relevance)} ({p.relevance}/5)', + f'- **Category:** {", ".join(p.categories)}', + f'- **Authors:** {p.author_text or "-"}', + f'- **Venue / Date:** {p.venue or "-"} / {p.publication_date or "-"}', + f'- **DOI:** {p.doi or "-"}', + f'- **Source:** {p.source}', + ] + + if p.ai_analysis_level: + level_text = { + 'full_text': 'Full text', + 'abstract': 'Abstract only', + }.get( + p.ai_analysis_level, + p.ai_analysis_level + ) + + lines.append( + f'- **AI Analysis:** {level_text}' + ) + + if p.oa_status: + lines.append( + f'- **OA Status:** {p.oa_status}' + ) + + if p.summary: + lines.append( + f'- **AI Summary:** {p.summary}' + ) + + elif p.abstract: + short = ( + p.abstract[:500] + + ('…' if len(p.abstract) > 500 else '') + ) + + lines.append( + f'- **Abstract excerpt:** {short}' + ) + + if p.ai_reason: + lines.append( + f'- **Why relevant:** {p.ai_reason}' + ) + + if p.pdf_url: + lines.append( + f'- **OA PDF:** {p.pdf_url}' + ) + + if p.url: + lines.append( + f'- **Link:** {p.url}' + ) + return '\n'.join(lines).strip()+'\n' + +def write_markdown(output_dir,filename,body): + p=Path(output_dir); p.mkdir(parents=True,exist_ok=True); out=p/filename; out.write_text(body,encoding='utf-8'); return str(out) diff --git a/src/paper_monitor/runner.py b/src/paper_monitor/runner.py new file mode 100644 index 0000000..c1c22c2 --- /dev/null +++ b/src/paper_monitor/runner.py @@ -0,0 +1,579 @@ +import argparse +import requests +import json +import logging + +from paper_monitor.circuit_breaker import protected_search +from collections import OrderedDict +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +from .config import load_config, env_value, resolve_path +from .db import PaperDB +from .util import paper_key, clean_text +from .scoring import classify_and_score +from .render import render_markdown, write_markdown +from .joplin import JoplinClient +from .ai_enrich import ( + enrich_with_gemini, + enrich_with_gemini_full_text, +) +from .collectors.ieee import ( + IEEECollector, + IEEEAPIError, +) +from .collectors.semantic_scholar import SemanticScholarCollector +from .collectors.crossref import CrossrefCollector +from .collectors.scholar_gmail import ScholarGmailCollector +from .oa_resolver import UnpaywallResolver +from .pdf_utils import download_and_extract_pdf + +log = logging.getLogger('paper_monitor') + + +def merge_paper(dst, src): + def longer(a, b): + return b if len(b or '') > len(a or '') else a + if not dst.doi and src.doi: + dst.doi = src.doi + dst.title = longer(dst.title, src.title) + if len(src.authors) > len(dst.authors): + dst.authors = src.authors + dst.abstract = longer(dst.abstract, src.abstract) + if not dst.url and src.url: + dst.url = src.url + dst.venue = longer(dst.venue, src.venue) + if not dst.publication_date and src.publication_date: + dst.publication_date = src.publication_date + if not dst.year and src.year: + dst.year = src.year + if src.citation_count is not None: + dst.citation_count = max(dst.citation_count or 0, src.citation_count) + if src.source and src.source not in dst.source: + dst.source = f'{dst.source} + {src.source}' if dst.source else src.source + return dst + + +def period(now, mode): + if mode == 'weekly': + iso = now.isocalendar() + return f'{iso.year}-W{iso.week:02d}' + return now.date().isoformat() + + +def _status_code(exc): + status = getattr(exc, 'status_code', None) + + if status is not None: + return status + + response = getattr(exc, 'response', None) + + if response is not None: + return response.status_code + + return None + + +def run(config_path, dry_run=False): + cfg = load_config(config_path) + app = cfg['app'] + search_cfg = cfg['search'] + sources = cfg.get('sources', {}) + logging.basicConfig( + level=getattr(logging, app.get('log_level', 'INFO').upper(), logging.INFO), + format='%(asctime)s %(levelname)s %(message)s', + ) + + now = datetime.now(ZoneInfo(app.get('timezone', 'Asia/Seoul'))) + local_date = now.date().isoformat() + lookback = int(app.get('lookback_days', 14)) + limit = int(app.get('max_papers_per_source_per_query', 50)) + queries = search_cfg.get('queries', []) + collected = [] + + if sources.get('ieee', {}).get('enabled'): + s = sources['ieee'] + c = IEEECollector(s['base_url'], env_value(s.get('api_key_env')), lookback, limit) + for q in queries: + try: + x = protected_search(c, q) + log.info('IEEE %r -> %d', q, len(x)) + collected += x + + except Exception as exc: + log.exception( + 'IEEE failed: %r', + q, + ) + + if isinstance( + exc, + IEEEAPIError, + ): + if exc.status_code is None: + log.error( + 'IEEE network/request error. ' + 'Stopping IEEE queries for this run.' + ) + else: + log.error( + 'IEEE HTTP %s. ' + 'Stopping IEEE queries for this run.', + exc.status_code, + ) + + break + + status = _status_code(exc) + + if status in ( + 401, + 403, + 429, + ): + log.error( + 'IEEE HTTP %s. ' + 'Stopping IEEE queries for this run.', + status, + ) + break + + log.error( + 'Unexpected IEEE error. ' + 'Stopping IEEE queries for this run.' + ) + break + + # Network problems + if isinstance( + exc, + ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ), + ): + log.error( + 'IEEE network error. ' + 'Stopping IEEE queries for this run.' + ) + break + + if sources.get('semantic_scholar', {}).get('enabled'): + s = sources['semantic_scholar'] + c = SemanticScholarCollector(s['base_url'], env_value(s.get('api_key_env')), lookback, limit) + for q in queries: + try: + x = protected_search(c, q) + log.info('Semantic Scholar %r -> %d', q, len(x)) + collected += x + except Exception as exc: + log.exception('Semantic Scholar failed: %r', q) + status = _status_code(exc) + # Authentication / authorization / rate-limit errors: + # Remaining queries are unlikely to succeed in this run. + if status in (401, 403, 429): + log.error( + 'Semantic Scholar HTTP %s. ' + 'Stopping Semantic Scholar queries for this run.', + status, + ) + break + # Network problem: + # Do not repeat the same timeout for every query. + if isinstance( + exc, + ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ), + ): + log.error( + 'Semantic Scholar network error. ' + 'Stopping Semantic Scholar queries for this run.' + ) + break + + if sources.get('crossref', {}).get('enabled'): + s = sources['crossref'] + c = CrossrefCollector(s['base_url'], env_value(s.get('mailto_env')), lookback, limit) + for q in queries: + try: + x = protected_search(c, q) + log.info('Crossref %r -> %d', q, len(x)) + collected += x + + except Exception as exc: + log.exception('Crossref failed: %r', q) + + status = _status_code(exc) + + # HTTP errors that are unlikely to recover during this run + if status in (401, 403, 429): + log.error( + 'Crossref HTTP %s. ' + 'Stopping Crossref queries for this run.', + status, + ) + break + + # Network problems + if isinstance( + exc, + ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ), + ): + log.error( + 'Crossref network error. ' + 'Stopping Crossref queries for this run.' + ) + break + + if sources.get('google_scholar_alert', {}).get('enabled'): + s = sources['google_scholar_alert'] + c = ScholarGmailCollector( + env_value(s.get('gmail_address_env')), + env_value(s.get('gmail_app_password_env')), + s.get('imap_host', 'imap.gmail.com'), + s.get('mailbox', 'INBOX'), + s.get('sender_contains', ''), + s.get('subject_contains', ''), + lookback, + ) + try: + x = c.collect() + log.info('Google Scholar Alert -> %d', len(x)) + collected += x + except Exception: + log.exception('Google Scholar Alert failed') + + merged = OrderedDict() + for p in collected: + p.title = clean_text(p.title) + if not p.title: + continue + k = paper_key(p.doi, p.title) + merged[k] = merge_paper(merged[k], p) if k in merged else p + + papers = [classify_and_score(p, search_cfg) for p in merged.values()] + papers = [p for p in papers if p.relevance >= int(app.get('min_relevance', 1))] + + db_path = resolve_path(cfg, app.get('database_path', './data/papers.db')) + db = PaperDB(db_path) + new = [] + + try: + # Determine which accepted papers are genuinely new before AI processing. + new_candidates = [ + p for p in papers + if not db.exists(p) + ] + + new_keys = { + paper_key(p.doi, p.title) + for p in new_candidates + } + + # Resolve Open Access information for new papers only. + oa = cfg.get('oa', {}) + + if oa.get('enabled'): + email = env_value( + oa.get('email_env', 'UNPAYWALL_EMAIL') + ) + + if not email: + log.warning( + 'Unpaywall email missing; ' + 'OA lookup skipped' + ) + + else: + resolver = UnpaywallResolver(email) + + oa_candidates = [ + p for p in new_candidates + if ( + p.doi + and p.doi.strip() + and not ( + p.pdf_url + and p.pdf_url.strip() + ) + ) + ] + + log.info( + 'Unpaywall candidates: %d / %d new papers', + len(oa_candidates), + len(new_candidates), + ) + + for p in oa_candidates: + try: + resolver.resolve(p) + + log.info( + 'Unpaywall resolved: status=%s pdf=%s title=%s', + p.oa_status or '-', + 'YES' if p.pdf_url else 'NO', + p.title, + ) + + except Exception: + # OA lookup failure must never stop paper collection. + log.exception( + 'Unpaywall lookup failed; continuing without OA information: %s', + p.title, + ) + + # ============================================================ + # 3. AI enrichment + # + # Candidates: + # + # A. Papers discovered for the first time in this run + # B. Papers from earlier runs whose ai_status == "failed" + # + # Historical papers with ai_status == "" are NOT automatically + # processed. This prevents unexpectedly processing the entire DB. + # ============================================================ + + + ai = cfg.get('ai', {}) + + if ai.get('enabled'): + api_key = env_value( + ai.get('api_key_env', 'GEMINI_API_KEY') + ) + + model = ai.get( + 'model', + 'gemini-3.6-flash', + ) + + max_papers = int( + ai.get('max_papers_per_run', 20) + ) + + # Retry only papers explicitly marked as failed. + # Existing historical papers with ai_status="" are NOT retried. + retry_candidates = db.list_ai_failed( + limit=max_papers + ) + + # Avoid processing the same paper twice if it is somehow + # present in both new_candidates and retry_candidates. + candidate_map = {} + + for p in new_candidates: + candidate_map[paper_key(p.doi, p.title)] = p + + for p in retry_candidates: + k = paper_key(p.doi, p.title) + + if k not in candidate_map: + candidate_map[k] = p + + ai_candidates = list( + candidate_map.values() + ) + + ai_candidates = sorted( + ai_candidates, + key=lambda x: x.relevance, + reverse=True, + )[:max_papers] + + if not api_key: + log.warning( + 'Gemini API key missing; AI enrichment skipped' + ) + + else: + log.info( + 'Gemini enrichment candidates: %d ' + '(new=%d retry=%d)', + len(ai_candidates), + len(new_candidates), + len(retry_candidates), + ) + + for p in ai_candidates: + full_text_done = False + abstract_done = False + had_failure = False + + # ----------------------------------------- + # 1. Prefer OA PDF full-text analysis + # ----------------------------------------- + if p.pdf_url and p.pdf_url.strip(): + try: + log.info( + 'Downloading OA PDF: %s', + p.title, + ) + + full_text = download_and_extract_pdf( + p.pdf_url + ) + + log.info( + 'PDF extracted: chars=%d title=%s', + len(full_text), + p.title, + ) + + enrich_with_gemini_full_text( + p, + full_text, + api_key, + model, + ) + + full_text_done = True + p.ai_status = 'done' + + log.info( + 'Gemini full-text enriched: ' + 'relevance=%d categories=%s title=%s', + p.relevance, + p.categories, + p.title, + ) + + except Exception: + had_failure = True + + log.exception( + 'Full-text AI failed; ' + 'trying abstract fallback: %s', + p.title, + ) + + # ----------------------------------------- + # 2. Abstract fallback + # ----------------------------------------- + if ( + not full_text_done + and p.abstract + and p.abstract.strip() + ): + try: + enrich_with_gemini( + p, + api_key, + model, + ) + + abstract_done = True + p.ai_status = 'done' + + log.info( + 'Gemini abstract enriched: ' + 'relevance=%d categories=%s title=%s', + p.relevance, + p.categories, + p.title, + ) + + except Exception: + had_failure = True + + log.exception( + 'Gemini abstract enrichment failed; ' + 'keeping original metadata: %s', + p.title, + ) + + # ----------------------------------------- + # 3. Final AI status + # ----------------------------------------- + if not full_text_done and not abstract_done: + if had_failure: + p.ai_status = 'failed' + + log.warning( + 'AI status=failed: %s', + p.title, + ) + + else: + p.ai_status = 'skipped' + + log.info( + 'AI status=skipped; ' + 'no usable PDF or abstract: %s', + p.title, + ) + + k = paper_key(p.doi, p.title) + + if k not in new_keys: + db.upsert( + p, + local_date=local_date, + ) + + # Store every accepted paper, regardless of AI success/failure. + for p in papers: + if db.upsert(p, local_date=local_date): + new.append(p) + + # Daily report is cumulative for the current local date, so reruns safely replace + # the same Joplin note without losing papers discovered earlier that day. + report_papers = db.list_first_seen_on(local_date) + + finally: + db.close() + + jcfg = cfg.get('joplin', {}) + suffix = period(now, jcfg.get('note_mode', 'daily')) + title = f'{jcfg.get("note_title_prefix", "Radar Literature")} - {suffix}' + body = render_markdown(report_papers, title, now) + out_dir = resolve_path(cfg, app.get('output_dir', './data/outbox')) + filename = title.replace('/', '-') + '.md' + out = write_markdown(out_dir, filename, body) + + note_id = '' + if jcfg.get('enabled') and not dry_run: + token = env_value(jcfg.get('token_env')) + if not token: + log.warning('Joplin token missing; Markdown only') + else: + try: + jc = JoplinClient(jcfg.get('base_url', 'http://127.0.0.1:41184'), token) + jc.ping() + folder = jc.ensure_folder_path(jcfg.get('notebook_path', ['Research', 'Radar Papers'])) + note_id = jc.create_or_update_note(title, body, folder, bool(jcfg.get('update_existing_note', True))) + for tag in jcfg.get('tags', []): + jc.add_tag_to_note(jc.ensure_tag(tag), note_id) + except Exception: + log.exception('Joplin failed; Markdown kept at %s', out) + + result = { + 'collected': len(collected), + 'merged': len(merged), + 'accepted': len(papers), + 'new': len(new), + 'report_count': len(report_papers), + 'report_title': title, + 'markdown': out, + 'joplin_note_id': note_id, + } + result_path = Path(resolve_path(cfg, './data/last_result.json')) + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') + log.info('Result: %s', result) + return result + + +def cli(): + p = argparse.ArgumentParser() + p.add_argument('--config', default='config.yaml') + p.add_argument('--dry-run', action='store_true') + a = p.parse_args() + run(a.config, a.dry_run) diff --git a/src/paper_monitor/scoring.py b/src/paper_monitor/scoring.py new file mode 100644 index 0000000..270af7c --- /dev/null +++ b/src/paper_monitor/scoring.py @@ -0,0 +1,6 @@ +def classify_and_score(paper, search_cfg): + hay=' '.join([paper.title,paper.abstract,paper.venue]).lower() + paper.categories=[cat for cat,terms in search_cfg.get('categories',{}).items() if any(str(t).lower() in hay for t in terms)] or ['Other'] + raw=sum(int(w) for term,w in search_cfg.get('relevance_terms',{}).items() if str(term).lower() in hay) + paper.relevance=5 if raw>=8 else 4 if raw>=5 else 3 if raw>=3 else 2 if raw>=1 else 1 + return paper diff --git a/src/paper_monitor/test_e2e_ai.py b/src/paper_monitor/test_e2e_ai.py new file mode 100644 index 0000000..0dc3982 --- /dev/null +++ b/src/paper_monitor/test_e2e_ai.py @@ -0,0 +1,119 @@ +import os +from datetime import datetime +from pathlib import Path + +from paper_monitor.ai_enrich import enrich_with_gemini_full_text +from paper_monitor.db import PaperDB +from paper_monitor.models import Paper +from paper_monitor.oa_resolver import UnpaywallResolver +from paper_monitor.pdf_utils import download_and_extract_pdf +from paper_monitor.render import render_markdown + + +def main(): + doi = "10.1371/journal.pone.0022647" + + paper = Paper( + title="E2E OA AI Test", + doi=doi, + source="test", + source_id="e2e-test", + ) + + print("1. Resolving OA PDF...") + + resolver = UnpaywallResolver( + os.environ["UNPAYWALL_EMAIL"] + ) + resolver.resolve(paper) + + print("oa_status =", paper.oa_status) + print("pdf_url =", paper.pdf_url) + + if not paper.pdf_url: + raise RuntimeError("OA PDF URL not found") + + print("2. Downloading and extracting PDF...") + + full_text = download_and_extract_pdf( + paper.pdf_url + ) + + print("text_length =", len(full_text)) + + print("3. Running Gemini full-text AI...") + + enrich_with_gemini_full_text( + paper, + full_text, + os.environ["GEMINI_API_KEY"], + "gemini-3.6-flash", + ) + + print("analysis_level =", paper.ai_analysis_level) + print("relevance =", paper.relevance) + print("categories =", paper.categories) + print("summary =", paper.summary) + print("reason =", paper.ai_reason) + + print("4. Saving to temporary DB...") + + db_path = "/tmp/paper_monitor_e2e.db" + + db = PaperDB(db_path) + + try: + db.upsert( + paper, + local_date="2026-08-15", + ) + + stored = db.conn.execute( + """ + SELECT + oa_status, + pdf_url, + ai_analysis_level, + relevance, + summary, + ai_reason + FROM papers + LIMIT 1 + """ + ).fetchone() + + print( + "db_analysis_level =", + stored["ai_analysis_level"], + ) + print( + "db_oa_status =", + stored["oa_status"], + ) + + finally: + db.close() + + print("5. Rendering Markdown...") + + body = render_markdown( + [paper], + "E2E AI Test", + datetime.now(), + ) + + output = Path("/tmp/e2e_ai_test.md") + output.write_text( + body, + encoding="utf-8", + ) + + print("markdown =", output) + print("") + print(body) + + print("E2E_TEST_OK") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/paper_monitor/test_retry_ai.py b/src/paper_monitor/test_retry_ai.py new file mode 100644 index 0000000..926d392 --- /dev/null +++ b/src/paper_monitor/test_retry_ai.py @@ -0,0 +1,168 @@ +import os + +from paper_monitor.ai_enrich import enrich_with_gemini_full_text +from paper_monitor.db import PaperDB +from paper_monitor.models import Paper +from paper_monitor.oa_resolver import UnpaywallResolver +from paper_monitor.pdf_utils import download_and_extract_pdf + + +def main(): + db_path = "/tmp/paper_monitor_retry_test.db" + + paper = Paper( + title="AI Retry Full Text Test", + doi="10.1371/journal.pone.0022647", + source="test", + source_id="retry-test", + ai_status="failed", + ) + + db = PaperDB(db_path) + + try: + print("1. Creating failed paper...") + + db.upsert( + paper, + local_date="2026-08-15", + ) + + failed = db.list_ai_failed(limit=10) + + print("failed_count =", len(failed)) + + if not failed: + raise RuntimeError( + "Failed paper was not returned by list_ai_failed()" + ) + + paper = failed[0] + + print("before_status =", paper.ai_status) + + print("2. Resolving OA PDF...") + + resolver = UnpaywallResolver( + os.environ["UNPAYWALL_EMAIL"] + ) + + resolver.resolve(paper) + + print("oa_status =", paper.oa_status) + print("pdf_url =", paper.pdf_url) + + if not paper.pdf_url: + raise RuntimeError( + "OA PDF URL not found" + ) + + print("3. Downloading and extracting PDF...") + + full_text = download_and_extract_pdf( + paper.pdf_url + ) + + print("text_length =", len(full_text)) + + print("4. Retrying Gemini...") + + enrich_with_gemini_full_text( + paper, + full_text, + os.environ["GEMINI_API_KEY"], + "gemini-3.6-flash", + ) + + paper.ai_status = "done" + + print( + "analysis_level =", + paper.ai_analysis_level, + ) + print( + "after_ai_status =", + paper.ai_status, + ) + print( + "relevance =", + paper.relevance, + ) + print( + "categories =", + paper.categories, + ) + + print("5. Updating existing DB row...") + + db.upsert( + paper, + local_date="2026-08-15", + ) + + row = db.conn.execute( + """ + SELECT + ai_status, + ai_analysis_level, + relevance, + summary, + ai_reason, + oa_status, + pdf_url + FROM papers + WHERE title=? + """, + (paper.title,), + ).fetchone() + + if not row: + raise RuntimeError( + "Updated paper not found in DB" + ) + + print( + "db_ai_status =", + row["ai_status"], + ) + print( + "db_analysis_level =", + row["ai_analysis_level"], + ) + print( + "db_relevance =", + row["relevance"], + ) + + if row["ai_status"] != "done": + raise RuntimeError( + "AI retry status was not saved as done" + ) + + if row["ai_analysis_level"] != "full_text": + raise RuntimeError( + "AI analysis level was not saved as full_text" + ) + + failed_after = db.list_ai_failed( + limit=10 + ) + + print( + "failed_count_after =", + len(failed_after), + ) + + if failed_after: + raise RuntimeError( + "Paper is still returned as failed after retry" + ) + + print("RETRY_TEST_OK") + + finally: + db.close() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/paper_monitor/util.py b/src/paper_monitor/util.py new file mode 100644 index 0000000..5c35178 --- /dev/null +++ b/src/paper_monitor/util.py @@ -0,0 +1,47 @@ +import hashlib, html, re +from datetime import datetime, timedelta, timezone +from dateutil import parser as dtparser + +DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) + +def clean_text(text): + if not text: return "" + text = html.unescape(str(text)) + text = re.sub(r"<[^>]+>", " ", text) + return re.sub(r"\s+", " ", text).strip() + +def normalize_doi(doi): + if not doi: return "" + doi = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", str(doi).strip(), flags=re.I) + doi = re.sub(r"^doi:\s*", "", doi, flags=re.I) + m = DOI_RE.search(doi) + return (m.group(0) if m else doi).rstrip(".,;").lower() + +def normalize_title(title): + return re.sub(r"[^a-z0-9가-힣]+", "", clean_text(title).lower()) + +def paper_key(doi, title): + d = normalize_doi(doi) + return "doi:" + d if d else "title:" + hashlib.sha256(normalize_title(title).encode()).hexdigest() + +def parse_date(value): + if not value: return "" + if isinstance(value, dict): + parts = value.get("date-parts", [[]])[0] + if parts: + y, m, d = parts[0], parts[1] if len(parts)>1 else 1, parts[2] if len(parts)>2 else 1 + return f"{y:04d}-{m:02d}-{d:02d}" + try: return dtparser.parse(str(value)).date().isoformat() + except Exception: return "" + +def within_lookback(date_str, days, now=None): + if not date_str: return True + now = now or datetime.now(timezone.utc) + try: return dtparser.parse(date_str).date() >= now.date() - timedelta(days=days) + except Exception: return True + +def md_escape(text): + return clean_text(text).replace("|", r"\|") + +def safe_url(url): + return (url or "").replace(" ", "%20").strip() diff --git a/state/provider_circuit_breaker_state.json b/state/provider_circuit_breaker_state.json new file mode 100644 index 0000000..ba018c7 --- /dev/null +++ b/state/provider_circuit_breaker_state.json @@ -0,0 +1,9 @@ +{ + "semantic_scholar": { + "open_until": 1786804590.1067948, + "reason": "http_403", + "strikes": { + "http_403": 1 + } + } +} \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..847b4b0 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,13 @@ +from datetime import datetime,timezone +from paper_monitor.models import Paper +from paper_monitor.util import normalize_doi,paper_key +from paper_monitor.scoring import classify_and_score +from paper_monitor.render import render_markdown + +def test_doi_normalization(): assert normalize_doi('https://doi.org/10.1109/ABC.123.')=='10.1109/abc.123' +def test_key_prefers_doi(): assert paper_key('10.1109/X.1','A title')=='doi:10.1109/x.1' +def test_scoring(): + p=Paper(title='Automotive Imaging Radar MIMO Antenna'); cfg={'categories':{'Antenna':['antenna'],'MIMO':['mimo']},'relevance_terms':{'automotive radar':3,'MIMO':2,'antenna':1}} + p=classify_and_score(p,cfg); assert 'Antenna' in p.categories; assert p.relevance>=3 +def test_render(): + md=render_markdown([Paper(title='Test | Paper',relevance=4,categories=['DOA'],url='https://example.com')],'Report',datetime.now(timezone.utc)); assert 'Test \\| Paper' in md; assert '★★★★' in md